Fester: a distributed build orchestrator with thermal-aware scheduling, live DAG, event replay, and failure analysis
This commit is contained in:
commit
0a9cb8fd97
|
|
@ -0,0 +1,64 @@
|
|||
# .github/funding.yml
|
||||
# ============================================================
|
||||
# GitHub Sponsors "Sponsor this project" button configuration.
|
||||
# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/displaying-a-sponsor-button-in-your-repository
|
||||
#
|
||||
# Each key below corresponds to a sponsored-developer account or platform
|
||||
# that will appear as a "Sponsor" button on the repo's main page.
|
||||
#
|
||||
# Replace the placeholder values with your actual usernames/addresses.
|
||||
# Comment out or delete any platforms you don't want to use.
|
||||
# ============================================================
|
||||
|
||||
# --- GitHub Sponsors ---
|
||||
# Your GitHub Sponsors username (the part after github.com/sponsors/)
|
||||
github: REPLACE_WITH_YOUR_GITHUB_USERNAME
|
||||
|
||||
# --- Patreon ---
|
||||
# Your Patreon username (the part after patreon.com/)
|
||||
patreon: REPLACE_WITH_YOUR_PATREON_USERNAME
|
||||
|
||||
# --- Open Collective ---
|
||||
# Your Open Collective slug (the part after opencollective.com/)
|
||||
open_collective: REPLACE_WITH_YOUR_OPENCOLLECTIVE_SLUG
|
||||
|
||||
# --- Ko-fi ---
|
||||
# Your Ko-fi username (the part after ko-fi.com/)
|
||||
ko_fi: REPLACE_WITH_YOUR_KOFI_USERNAME
|
||||
|
||||
# --- Tidelift ---
|
||||
# Your Tidelift package name (e.g. "npm/foo" or "pypi/bar")
|
||||
tidelift: REPLACE_WITH_YOUR_TIDELIFT_PACKAGE
|
||||
|
||||
# --- CommunityBridge ---
|
||||
# Your CommunityBridge project name
|
||||
community_bridge: REPLACE_WITH_YOUR_COMMUNITYBRIDGE_PROJECT
|
||||
|
||||
# --- Liberapay ---
|
||||
# Your Liberapay username (the part after liberapay.com/)
|
||||
liberapay: REPLACE_WITH_YOUR_LIBERAPAY_USERNAME
|
||||
|
||||
# --- IssueHunt ---
|
||||
# Your IssueHunt username
|
||||
issuehunt: REPLACE_WITH_YOUR_ISSUEHUNT_USERNAME
|
||||
|
||||
# --- LFX Crowdfunding ---
|
||||
# Your LFX project name
|
||||
lfx_crowdfunding: REPLACE_WITH_YOUR_LFX_PROJECT
|
||||
|
||||
# --- Polar ---
|
||||
# Your Polar repository (e.g. "owner/repo")
|
||||
polar: REPLACE_WITH_YOUR_GITHUB_ORG/fester
|
||||
|
||||
# --- Custom platforms / direct links ---
|
||||
# A list of up to 4 custom {label, url} entries that appear as buttons.
|
||||
# Use these for crypto donation links, your own donate page, etc.
|
||||
custom:
|
||||
- label: "Donate"
|
||||
url: "https://git.dcos.net/dcosnet/fester/src/branch/main/donate.md"
|
||||
# - label: "Bitcoin"
|
||||
# url: "https://your-blockchain-explorer/address/YOUR_BTC_ADDRESS"
|
||||
# - label: "Monero"
|
||||
# url: "https://your-xmr-explorer/address/YOUR_XMR_ADDRESS"
|
||||
# - label: "PayPal"
|
||||
# url: "https://www.paypal.com/paypalme/YOUR_PAYPAL_ME"
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
# ============================================================
|
||||
# Fester .gitignore
|
||||
# ============================================================
|
||||
|
||||
# ---- Python ----
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# ---- Virtual environments ----
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
.python-version
|
||||
|
||||
# ---- Testing / coverage ----
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
|
||||
# ---- Fester runtime data ----
|
||||
*.db
|
||||
*.db-journal
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
*.sqlite
|
||||
*.sqlite-journal
|
||||
*.sqlite-wal
|
||||
*.sqlite-shm
|
||||
|
||||
# Fester storage dirs (local dev)
|
||||
.fester/
|
||||
fester-data/
|
||||
/var/lib/fester/
|
||||
snapshots/
|
||||
cas/
|
||||
cache/
|
||||
*.qcow2
|
||||
*.tar.gz
|
||||
|
||||
# Config with secrets (keep config.yaml.example in git, ignore actual config)
|
||||
config.local.yaml
|
||||
config.runtime.yaml
|
||||
storage.json
|
||||
node_roles.json
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# ---- Logs ----
|
||||
*.log
|
||||
logs/
|
||||
fester.log
|
||||
fester_backend.log
|
||||
mock_agent.log
|
||||
|
||||
# ---- IDE / Editor ----
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.project
|
||||
.classpath
|
||||
.settings/
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
.atom/
|
||||
.brackets.json
|
||||
|
||||
# ---- OS ----
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
desktop.ini
|
||||
|
||||
# ---- Docker ----
|
||||
docker-compose.override.yml
|
||||
|
||||
# ---- Node (if you ever add a JS build step) ----
|
||||
node_modules/
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
|
||||
# ---- Build artifacts ----
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
*.dylib
|
||||
build.tar.gz
|
||||
build/
|
||||
|
||||
# ---- Screenshots (keep in git, but ignore ad-hoc captures) ----
|
||||
# docs/screenshots/ is tracked — these are ad-hoc:
|
||||
/tmp/
|
||||
*.tmp
|
||||
|
||||
# ---- Misc ----
|
||||
*.bak
|
||||
*.orig
|
||||
*.rej
|
||||
*.diff
|
||||
*.patch
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
# Changelog
|
||||
|
||||
All notable changes to Fester are documented here.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- Documentation: README.md, quickstart.md, CONTRIBUTING.md, CHANGELOG.md
|
||||
- Tooling: bootstrap.sh, run.sh, pyproject.toml, .gitignore
|
||||
- Docker support (Dockerfile)
|
||||
- Screenshots for all 8 UI pages
|
||||
|
||||
## [0.3.0] — 2026-06-28
|
||||
|
||||
### Added — Storage layer
|
||||
- SQLite-backed persistence for builds, sessions, events, and node states
|
||||
- Btrfs CoW reflink snapshots (`backend/storage/btrfs_cas.py`)
|
||||
- QCOW2 workspace freezing with real mount + rsync (`backend/storage/qcow2_freeze.py`)
|
||||
- tmpfs workspace acceleration
|
||||
- Storage status + config + freeze + reflink + snapshots endpoints
|
||||
|
||||
### Added — tmux integration (E9)
|
||||
- TmuxManager with session create/list/inspect/capture-output/kill
|
||||
- `runtime: "tmux"` action runtime — runs actions in detached tmux sessions
|
||||
- Live output viewer in Sessions page UI
|
||||
- `/api/actions/active`, `/api/actions/{name}/tmux/output`, `/tmux/kill` endpoints
|
||||
|
||||
### Added — mosh/ssh shell attach (E10)
|
||||
- MoshManager with command builder (mosh-via-SSH-bootstrap)
|
||||
- `/api/nodes/{name}/shell` GET + POST endpoints
|
||||
- "⌘ Shell" button on Dashboard node rows (copies command to clipboard)
|
||||
|
||||
### Added — Real engine wiring
|
||||
- BuildRunner wraps PipelineEngine with build lifecycle + persistence
|
||||
- Real DAG execution with proper cwd + env passing
|
||||
- Failure propagation: skipped actions on failed deps
|
||||
- Critical path computation + tagging in events
|
||||
- Debugger pause/resume/step actually controls running engines
|
||||
|
||||
### Added — Cause graph + blast radius
|
||||
- `/api/propagation/{action}` — forward blast radius computation
|
||||
- `/api/cause/explain/{node}` — causal chain for a node
|
||||
- Blast Radius panel in Cause Graph UI page
|
||||
- Failure autopsy with proper session ID correlation
|
||||
|
||||
### Added — Storage + target catalog
|
||||
- `/api/targets` reads from `backend/targets/catalog.py` (6 systems: Gentoo, Buildroot, OpenWrt, ALFS, SourceMage, Lunar)
|
||||
- `/api/targets/arches` + `/api/targets/runtimes` endpoints
|
||||
|
||||
### Added — Health + metrics
|
||||
- `/api/health` endpoint (version, bus_subscribers, ws_clients, builds_known, timeline_events)
|
||||
- Live health indicator in topbar (all pages)
|
||||
- Prometheus metrics aligned with grafana config names (`fester_node_cpu`, `fester_pipeline_actions_total`, etc.)
|
||||
- Pipeline action counters + cache hit counters wired via bus subscriber
|
||||
|
||||
### Added — UI enhancements
|
||||
- New Timeline page (per-node event drill-down)
|
||||
- Node policy dropdowns (preferred/avoid/neutral) on Dashboard
|
||||
- Per-node "Probe" buttons + "Probe All" button
|
||||
- Build cancel + replay buttons on Sessions page
|
||||
- Event filter chips on Dashboard (toggle by event type)
|
||||
|
||||
### Added — CLI overhaul
|
||||
- 35+ subcommands (was 11)
|
||||
- Nested subcommands: `node {list,set-policy,probe,probe-all}`, `cause {explain,trace,events}`, `replay {start,step,reset}`, `debugger {state,pause,resume,step}`
|
||||
- New: `build`, `builds`, `build-info`, `cancel`, `release`, `targets`, `toggle-target`, `blast`, `timeline`, `rewind`, `autopsy`, `policy-list`, `health`
|
||||
- `--watch` flag on `build` command (streams events via WebSocket)
|
||||
|
||||
### Fixed — Critical backend bugs
|
||||
- `api/api.py` order bug (`bus.subscribe()` before `bus = EventBus()`)
|
||||
- 7 broken `_endpoint` symbol imports
|
||||
- `cause_graph.attach_bus` + `emit_debug` defined at module scope (zero indentation)
|
||||
- Three missing files: `timeline_store.py`, `failure_autopsy.py`, `failure_propagation.py`
|
||||
- Two Python module/package name collisions (`scheduler.py` vs `scheduler/`, `nodes.py` vs `nodes/`, `cache.py` vs `cache/`)
|
||||
- `execute_action` passing empty `{}` as cwd → `TypeError`
|
||||
- Build env never reaching subprocess
|
||||
- Debugger/pipeline-control `_ENGINE` always `None`
|
||||
- `failure_propagation` looking for `event.data.deps` (legacy shape)
|
||||
- Autopsy session ID mismatch
|
||||
- `/api/release` was a pure stub
|
||||
- `/api/policy/set` didn't persist
|
||||
- `/ws-debugger` returned canned data, ignored commands
|
||||
- `forgejo.on_push_event` wrong PipelineEngine arity
|
||||
- `pipeline/feedback.py` PolicyEngine() with no args
|
||||
|
||||
### Fixed — Standardization
|
||||
- ~20 bare imports converted to `backend.*` prefix
|
||||
- Two parallel event taxonomies unified (`types.py` re-exports from `schema.py`)
|
||||
- `MinioCache.__init__` made lazy (was crashing on import if MinIO unreachable)
|
||||
- `cache.py` graceful fallback for non-writable paths
|
||||
- `roles_store.py` graceful fallback for non-writable `/etc/fester/`
|
||||
- `FesterEvent` made `node`/`action`/`state` truly optional (had `Optional[str]` type but no default)
|
||||
|
||||
## [0.2.0] — 2026-06-27
|
||||
|
||||
### Added — Real FastAPI backend
|
||||
- `backend/main.py` — FastAPI app with 50+ routes
|
||||
- All routers wired: replay, autopsy, timeline, debugger, pipeline_control, nodes, metrics, cause
|
||||
- WebSocket hub with `/ws`, `/ws-targets`, `/ws-debugger`
|
||||
- Ambient loop for node probing + drift
|
||||
- SQLite storage layer (`backend/storage/sqlite_db.py`)
|
||||
- Node probe module (`backend/nodes/probe.py`)
|
||||
- BuildRunner (`backend/pipeline/runner.py`)
|
||||
|
||||
### Added — UI overhaul (7 pages)
|
||||
- Dashboard (`index.html`) — 3-column layout with cluster stats, live stream, quick actions
|
||||
- Live DAG (`ui/live_dag.html`) — layered Sugiyama-lite layout, SVG edges, click-to-inspect
|
||||
- Replay (`ui/replay.html`) — timeline scrubber, play/pause/step, structured inspector
|
||||
- Sessions (`ui/sessions.html`) — build history + replay links
|
||||
- Metrics (`ui/metrics.html`) — live charts + per-node cards
|
||||
- Cause Graph (`ui/cause.html`) — radial graph visualization
|
||||
- Debugger (`ui/debugger.html`) — step-through execution control
|
||||
|
||||
### Added — Shared infrastructure
|
||||
- `style.css` — full design system (dark theme tokens, panels, badges, buttons, scrollbars)
|
||||
- `ui/app.js` — shared shell (WS auto-reconnect, FesterTargets global, toasts, topbar nav)
|
||||
- `cockpit/fester-module/targets.js` — fixed to expose `FesterTargets` global
|
||||
|
||||
### Fixed
|
||||
- `live_dag.html` was broken (referenced undefined `FesterTargets`)
|
||||
- `style.css` was 0 bytes
|
||||
- `index.html` was a 30-line throwaway
|
||||
|
||||
## [0.1.0] — Initial commit
|
||||
|
||||
### Added
|
||||
- Basic project structure
|
||||
- Backend modules (many stubbed)
|
||||
- Cockpit module
|
||||
- Install scripts
|
||||
- CHEATSHEET.md
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
# Fester Cluster Build System — Cheatsheet
|
||||
|
||||
## 🧠 Core Concept
|
||||
|
||||
A distributed, DAG-driven build execution system with:
|
||||
|
||||
- real-time scheduling
|
||||
- node-aware load balancing
|
||||
- thermal + policy constraints
|
||||
- cache-aware execution
|
||||
- deterministic replay/debugging
|
||||
|
||||
---
|
||||
|
||||
## 🧩 System Components
|
||||
|
||||
### 1. Scheduler
|
||||
Chooses best node per action using:
|
||||
|
||||
- CPU load
|
||||
- temperature
|
||||
- policy rules
|
||||
- historical intelligence feedback
|
||||
|
||||
---
|
||||
|
||||
### 2. Pipeline Engine
|
||||
Executes DAG actions sequentially or step-debugged.
|
||||
|
||||
Supports:
|
||||
- live execution
|
||||
- interactive stepping
|
||||
- replay mode
|
||||
|
||||
---
|
||||
|
||||
### 3. Timeline Store
|
||||
Immutable event log of entire system.
|
||||
|
||||
Used for:
|
||||
- replay
|
||||
- debugging
|
||||
- autopsy analysis
|
||||
|
||||
---
|
||||
|
||||
### 4. Failure System
|
||||
Detects and propagates failure:
|
||||
|
||||
- backward → root cause candidates
|
||||
- forward → downstream impact
|
||||
|
||||
---
|
||||
|
||||
### 5. Critical Path Analyzer
|
||||
Identifies bottleneck chain in DAG execution.
|
||||
|
||||
---
|
||||
|
||||
### 6. Cache Layer
|
||||
Supports:
|
||||
- distributed artifact reuse
|
||||
- MinIO backend
|
||||
- future: Btrfs/QCOW2 snapshot acceleration
|
||||
|
||||
---
|
||||
|
||||
### 7. Debugger Mode
|
||||
Interactive execution control:
|
||||
|
||||
- pause
|
||||
- step
|
||||
- resume
|
||||
- scheduler preview before execution
|
||||
|
||||
---
|
||||
|
||||
## 🧭 Execution Flow
|
||||
|
||||
1. Build graph generated from spec
|
||||
2. Scheduler selects node per action
|
||||
3. Timeline records decision
|
||||
4. Executor runs action on node
|
||||
5. Events streamed to UI
|
||||
6. Failures propagate through DAG
|
||||
7. Optional debugger intercepts execution
|
||||
|
||||
---
|
||||
|
||||
## 📡 Event Types
|
||||
|
||||
- `node`
|
||||
- `pipeline`
|
||||
- `schedule_decision`
|
||||
- `failure_propagation`
|
||||
- `debugger_preview`
|
||||
|
||||
---
|
||||
|
||||
## 🖥 UI Modes
|
||||
|
||||
### Live Mode
|
||||
- real-time DAG rendering
|
||||
- node state updates
|
||||
|
||||
### Debug Mode
|
||||
- step execution
|
||||
- scheduler inspection
|
||||
- manual control
|
||||
|
||||
### Autopsy Mode
|
||||
- failure root cause analysis
|
||||
- dependency tracing
|
||||
- critical path overlay
|
||||
|
||||
---
|
||||
|
||||
## ⚙️ Node Selection Model
|
||||
|
||||
Score-based weighted system:
|
||||
|
||||
- CPU availability
|
||||
- thermal headroom
|
||||
- policy constraints
|
||||
- historical instability penalties
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Execution Backends (planned)
|
||||
|
||||
- distcc
|
||||
- LXC
|
||||
- libvirt
|
||||
- native execution
|
||||
- cross-compilation toolchains
|
||||
|
||||
---
|
||||
|
||||
## 📦 Cache Strategy
|
||||
|
||||
- local ccache
|
||||
- shared MinIO cache
|
||||
- future: Btrfs/QCOW2 snapshot acceleration
|
||||
|
||||
---
|
||||
|
||||
## 🔐 Design Principle
|
||||
|
||||
> The system must always be replayable, explainable, and deterministic.
|
||||
|
||||
No hidden state.
|
||||
No opaque scheduling.
|
||||
Everything is observable.
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
# Contributing to Fester
|
||||
|
||||
Thanks for your interest in improving Fester! This document covers the basics.
|
||||
|
||||
## 🚀 Quick Start for Contributors
|
||||
|
||||
```bash
|
||||
# 1. Fork + clone
|
||||
git clone https://git.dcos.net/dcosnet/fester.git
|
||||
cd fester
|
||||
|
||||
# 2. Bootstrap dev environment (creates venv, installs deps + dev tools)
|
||||
./bootstrap.sh --dev
|
||||
|
||||
# 3. Run the backend in dev mode (auto-reload on changes)
|
||||
./run.sh --dev
|
||||
|
||||
# 4. Run tests (when we have them)
|
||||
pytest
|
||||
|
||||
# 5. Lint + format
|
||||
ruff check .
|
||||
ruff format .
|
||||
```
|
||||
|
||||
## 🏗️ Architecture Overview
|
||||
|
||||
Read **[README.md](README.md)** first — it has the full architecture diagram and project layout.
|
||||
|
||||
The key insight: Fester has a strict separation between:
|
||||
- **EventBus** — the singleton message bus (all events flow through it)
|
||||
- **Subscribers** — bus listeners that index/journal/broadcast events (TimelineStore, cause_graph, FailurePropagator, the WS broadcaster)
|
||||
- **Routers** — FastAPI APIRouters under `backend/api/` that expose REST endpoints
|
||||
- **UI** — vanilla HTML/JS pages under `ui/` that subscribe to the WS stream and call REST endpoints
|
||||
|
||||
When you add a new feature, ask:
|
||||
1. Does this emit events? → Use `bus.emit(EventType.X, ...)` from `backend/events/schema.py`
|
||||
2. Does this need persistence? → Add a method to `backend/storage/sqlite_db.py:Storage`
|
||||
3. Does this need a UI? → Add a page under `ui/` and a nav entry in `ui/app.js:Fester.mountShell()`
|
||||
4. Does this need a CLI command? → Add a subcommand to `cli/fester.py`
|
||||
|
||||
## 📝 Coding Standards
|
||||
|
||||
### Python
|
||||
|
||||
- **Python 3.12+** — use modern syntax (match statements, type hints, etc.)
|
||||
- **Type hints** — add them to new code; don't worry about retrofitting old code
|
||||
- **Imports** — use `from backend.X.Y import Z` (not bare `from X.Y import Z`)
|
||||
- **EventBus** — always emit events via `bus.emit()`, never mutate state directly from a request handler
|
||||
- **Pydantic** — use `BaseModel` for request bodies in routers
|
||||
- **Async** — use `async def` for endpoints that touch the DB or do I/O
|
||||
|
||||
### JavaScript (UI)
|
||||
|
||||
- **Vanilla JS** — no frameworks (React, Vue, etc.). Keep it dependency-free.
|
||||
- **Shared shell** — every page calls `Fester.mountShell('page-id')` to get the topbar + WS connection
|
||||
- **API calls** — use `Fester.api(path, opts)` (returns parsed JSON)
|
||||
- **Toasts** — use `Fester.toast(msg, kind)` for user notifications
|
||||
- **No build step** — pages load directly via `<script src="/ui/app.js">`
|
||||
|
||||
### CSS
|
||||
|
||||
- **Design tokens** — use `var(--token)` (defined in `style.css`). Don't hardcode colors.
|
||||
- **Dark theme only** — no light mode. Tokens are already dark.
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
pytest
|
||||
|
||||
# Run a specific test file
|
||||
pytest tests/test_pipeline.py
|
||||
|
||||
# Run with coverage
|
||||
pytest --cov=backend --cov-report=html
|
||||
|
||||
# Run only fast tests
|
||||
pytest -m "not slow"
|
||||
```
|
||||
|
||||
Tests live in `tests/` (we don't have many yet — contributions welcome!).
|
||||
|
||||
## 🔍 Code Quality
|
||||
|
||||
```bash
|
||||
# Lint
|
||||
ruff check .
|
||||
|
||||
# Format
|
||||
ruff format .
|
||||
|
||||
# Type check
|
||||
mypy backend/
|
||||
|
||||
# Check for syntax errors in inline <script> blocks
|
||||
node scripts/check_syntax.js ui/*.html index.html
|
||||
```
|
||||
|
||||
## 📦 Commit Conventions
|
||||
|
||||
Use [Conventional Commits](https://www.conventionalcommits.org/):
|
||||
|
||||
```
|
||||
feat: add tmux output viewer to sessions page
|
||||
fix: pass real cwd to execute_action instead of empty dict
|
||||
docs: add quickstart guide
|
||||
chore: update dependencies
|
||||
refactor: split BuildRunner from PipelineEngine
|
||||
test: add tests for failure_propagation
|
||||
```
|
||||
|
||||
## 🐛 Bug Reports
|
||||
|
||||
When filing an issue, include:
|
||||
1. Fester version (`fester health` output)
|
||||
2. Python version (`python3 --version`)
|
||||
3. OS + distro
|
||||
4. Steps to reproduce
|
||||
5. Expected vs actual behavior
|
||||
6. Relevant log output (`/tmp/fester_backend.log`)
|
||||
7. Screenshots if UI-related
|
||||
|
||||
## ✨ Pull Requests
|
||||
|
||||
1. Fork the repo + create a branch: `git checkout -b feat/my-feature`
|
||||
2. Make your changes
|
||||
3. Run `ruff check . && ruff format .` before committing
|
||||
4. Write a clear commit message (Conventional Commits)
|
||||
5. Open a PR with:
|
||||
- What changed + why
|
||||
- Screenshots (if UI changes)
|
||||
- Test results
|
||||
- Any breaking changes
|
||||
|
||||
## 🆘 Getting Help
|
||||
|
||||
- Read **[CHEATSHEET.md](CHEATSHEET.md)** for the operator survival guide
|
||||
- Read **[quickstart.md](quickstart.md)** for getting started
|
||||
- Browse the **[API docs](http://localhost:8080/docs)** at runtime
|
||||
- Check existing issues: https://git.dcos.net/dcosnet/fester/issues
|
||||
|
||||
## 📜 License
|
||||
|
||||
By contributing, you agree that your contributions will be licensed under the AGPL-3.0 — see **[LICENSE](LICENSE)**.
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
🧾 SOFTWARE PROVIDED “AS IS” CLAUSE
|
||||
📄 Disclaimer of Warranty and Liability
|
||||
|
||||
This software is provided “as is”, without warranty of any kind, express or implied. This includes, but is not limited to, warranties of merchantability, fitness for a particular purpose, non-infringement, and any warranties arising from course of dealing or usage of trade.
|
||||
|
||||
The authors and contributors make no representation or guarantee that:
|
||||
|
||||
the software will function uninterrupted or error-free,
|
||||
builds, scheduling decisions, or execution results will be correct or deterministic,
|
||||
distributed execution across heterogeneous nodes will be stable or safe in all environments,
|
||||
caching, snapshotting, or replay features will not result in data loss or corruption,
|
||||
integrations (including but not limited to libvirt, LXC, tmux, mosh, distcc, MinIO, Prometheus, Grafana, Forgejo, or any external toolchain) will behave as expected in all configurations.
|
||||
⚙️ Operational Risk Acknowledgement
|
||||
|
||||
This system is capable of:
|
||||
|
||||
executing distributed code compilation,
|
||||
scheduling workloads across physical and virtual nodes,
|
||||
modifying system-level environments and build artifacts,
|
||||
managing cache layers, snapshots, and execution state.
|
||||
|
||||
You acknowledge that:
|
||||
|
||||
misconfiguration may result in system instability, data loss, or resource exhaustion,
|
||||
execution across remote nodes may cause unpredictable behavior under load,
|
||||
performance optimization logic may prioritize throughput over safety unless explicitly configured otherwise.
|
||||
🔒 No Liability
|
||||
|
||||
In no event shall the authors, maintainers, or contributors be liable for any:
|
||||
|
||||
direct, indirect, incidental, or consequential damages,
|
||||
loss of data, revenue, or system availability,
|
||||
hardware degradation or thermal damage,
|
||||
build failures or deployment issues,
|
||||
security vulnerabilities arising from system configuration or integration.
|
||||
🧠 Responsibility of Use
|
||||
|
||||
You are solely responsible for:
|
||||
|
||||
validating build targets and toolchains,
|
||||
verifying scheduling and execution policies,
|
||||
ensuring safe node configuration and thermal limits,
|
||||
reviewing cache and snapshot behavior before production use,
|
||||
controlling access to the system and its APIs.
|
||||
⚡ High-Risk System Notice
|
||||
|
||||
This software operates in a high-concurrency, distributed execution domain. It is not intended for safety-critical systems unless independently audited and hardened for such use cases.
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
# Dockerfile for Fester
|
||||
# ============================================================
|
||||
# Multi-stage build: installs Python deps, copies code, runs uvicorn.
|
||||
#
|
||||
# Build:
|
||||
# docker build -t fester .
|
||||
#
|
||||
# Run:
|
||||
# docker run -p 8080:8080 \
|
||||
# -v fester-db:/var/lib/fester \
|
||||
# -v fester-config:/etc/fester \
|
||||
# fester
|
||||
#
|
||||
# Or with docker-compose (see docker-compose.yml):
|
||||
# docker-compose up
|
||||
# ============================================================
|
||||
|
||||
FROM python:3.12-slim AS base
|
||||
|
||||
# Install system dependencies (optional tools activate features when present)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tmux \
|
||||
mosh \
|
||||
openssh-client \
|
||||
qemu-utils \
|
||||
rsync \
|
||||
btrfs-progs \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create fester user + directories
|
||||
RUN useradd -m -u 1000 fester && \
|
||||
mkdir -p /var/lib/fester/cas /var/lib/fester/snapshots /var/lib/fester/cache /etc/fester && \
|
||||
chown -R fester:fester /var/lib/fester /etc/fester
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy pyproject.toml first (better layer caching)
|
||||
COPY pyproject.toml ./
|
||||
|
||||
# Install Python dependencies
|
||||
RUN pip install --no-cache-dir \
|
||||
"fastapi>=0.128" \
|
||||
"uvicorn[standard]>=0.30" \
|
||||
"websockets>=12" \
|
||||
"pydantic>=2" \
|
||||
"aiohttp>=3.9" \
|
||||
"minio>=7.2" \
|
||||
"prometheus-client>=0.20" \
|
||||
"PyYAML>=6" \
|
||||
"requests>=2.31" \
|
||||
"websocket-client>=1.7"
|
||||
|
||||
# Copy the rest of the code
|
||||
COPY --chown=fester:fester . .
|
||||
|
||||
# Make scripts executable
|
||||
RUN chmod +x run.sh cli/fester.py 2>/dev/null || true
|
||||
|
||||
# Switch to non-root user
|
||||
USER fester
|
||||
|
||||
# Default config (can be overridden via volume mount at /etc/fester/config.yaml)
|
||||
ENV FESTER_DB_PATH=/var/lib/fester/fester.db
|
||||
ENV FESTER_CONFIG=/app/config.yaml
|
||||
ENV FESTER_NO_DRIFT=0
|
||||
ENV FESTER_AUTOBUILD=0
|
||||
|
||||
# Expose the backend port
|
||||
EXPOSE 8080
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD curl -f http://localhost:8080/api/health || exit 1
|
||||
|
||||
# Run the backend
|
||||
CMD ["python3", "-m", "uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
|
|
@ -0,0 +1,679 @@
|
|||
========================================================================
|
||||
PROJECT: fester
|
||||
COPYRIGHT: Copyright (C) 2026 dcos.net
|
||||
HOMEPAGE: https://git.dcos.net/dcosnet/fester/
|
||||
REPOSITORY: https://git.dcos.net/dcosnet/fester/
|
||||
LICENSE: GNU Affero General Public License v3.0 (AGPL-3.0)
|
||||
========================================================================
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------
|
||||
APPENDIX: HOW TO APPLY THESE TERMS TO YOUR NEW PROGRAMS
|
||||
|
||||
To ensure your repository complies with the remote network interaction
|
||||
requirements (Section 13 of the AGPLv3), you must ensure users
|
||||
interacting with your stack can access the source code.
|
||||
|
||||
This project, fester, complies by maintaining its primary development
|
||||
repository at https://git.dcos.net. Any derivative works must retain
|
||||
this notice and provide equivalent access to source code.
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
🟩 2. CHEATSHEET.md (Operator Survival Guide)
|
||||
# 🧠 Fester Operator Cheatsheet
|
||||
|
||||
Distributed Build + Scheduler + Observability System
|
||||
|
||||
---
|
||||
|
||||
# 🚀 STARTING A BUILD
|
||||
|
||||
```bash
|
||||
fester build ./project.yaml
|
||||
🧩 VIEW LIVE SYSTEM
|
||||
Cockpit Module: Fester
|
||||
Live DAG: /ui/live_dag.html
|
||||
Replay View: /ui/replay.html
|
||||
📡 KEY CONCEPTS
|
||||
Nodes
|
||||
|
||||
Machines participating in builds (physical, VM, or container)
|
||||
|
||||
Targets
|
||||
|
||||
Compilation environments:
|
||||
|
||||
x86_64-linux-gnu
|
||||
aarch64-linux-gnu
|
||||
riscv64
|
||||
embedded/toolchain targets
|
||||
Actions
|
||||
|
||||
Graph nodes representing build steps
|
||||
|
||||
Scheduler
|
||||
|
||||
Chooses best node based on:
|
||||
|
||||
load
|
||||
temperature
|
||||
policy rules
|
||||
historical performance
|
||||
🧠 DEBUGGING
|
||||
Failure Autopsy
|
||||
GET /api/autopsy/<build_id>
|
||||
Timeline Replay
|
||||
GET /api/replay/<session_id>
|
||||
📦 CACHE SYSTEM
|
||||
|
||||
Supports:
|
||||
|
||||
ccache
|
||||
MinIO distributed cache
|
||||
Btrfs snapshot cache
|
||||
tmpfs acceleration layer
|
||||
🧊 SNAPSHOTS
|
||||
|
||||
Freeze state:
|
||||
|
||||
qcow2 image snapshots
|
||||
Btrfs copy-on-write states
|
||||
⚙️ NODE CONTROL
|
||||
fester node list
|
||||
fester node set-policy <node> preferred
|
||||
fester node set-policy <node> avoid
|
||||
🔥 SCHEDULER MODES
|
||||
unified (default)
|
||||
weighted thermal-aware
|
||||
cache-first
|
||||
target-isolated
|
||||
experimental intelligence mode
|
||||
⚠️ SAFETY NOTES
|
||||
Do NOT run unrestricted builds on production nodes
|
||||
Monitor thermal load in Grafana
|
||||
Ensure cache integrity for distributed builds
|
||||
Validate cross-compile toolchains before enabling targets
|
||||
🧠 DESIGN PRINCIPLE
|
||||
|
||||
"Every build is reproducible, explainable, and replayable."
|
||||
|
||||
|
||||
---
|
||||
|
||||
# 🟩 3. `PRODUCTION_HARDENING.md`
|
||||
|
||||
```markdown
|
||||
# 🛡️ Fester Production Hardening Guide
|
||||
|
||||
---
|
||||
|
||||
# 🌡️ 1. THERMAL SAFETY
|
||||
|
||||
## Required
|
||||
- Enable node temperature monitoring
|
||||
- Set max thermal threshold per node
|
||||
|
||||
## Recommended
|
||||
- throttle scheduler above 85°C
|
||||
- disable high-parallel builds under sustained load
|
||||
|
||||
---
|
||||
|
||||
# 🧠 2. SCHEDULER SAFETY
|
||||
|
||||
Avoid:
|
||||
- unrestricted global scheduling in mixed architectures
|
||||
- running full cluster on single target policy
|
||||
|
||||
Enable:
|
||||
- weighted scheduler
|
||||
- per-node policy constraints
|
||||
- cache-aware routing
|
||||
|
||||
---
|
||||
|
||||
# 📦 3. CACHE INTEGRITY
|
||||
|
||||
Use:
|
||||
- checksum validation for artifacts
|
||||
- MinIO redundancy if cluster > 3 nodes
|
||||
- avoid mixing tmpfs + persistent cache without sync barriers
|
||||
|
||||
---
|
||||
|
||||
# 🧊 4. SNAPSHOT SAFETY
|
||||
|
||||
If using:
|
||||
- qcow2 snapshots
|
||||
- Btrfs CoW layers
|
||||
|
||||
Ensure:
|
||||
- periodic snapshot compaction
|
||||
- rollback testing before production builds
|
||||
|
||||
---
|
||||
|
||||
# 🧩 5. CROSS-COMPILATION RISKS
|
||||
|
||||
- toolchain mismatch = silent binary corruption risk
|
||||
- always validate ELF output per target arch
|
||||
- isolate toolchains per execution environment
|
||||
|
||||
---
|
||||
|
||||
# 🌐 6. DISTRIBUTED EXECUTION
|
||||
|
||||
- ensure clock sync (NTP mandatory)
|
||||
- avoid partial node visibility during scheduling
|
||||
- handle node dropouts as first-class events
|
||||
|
||||
---
|
||||
|
||||
# 📊 7. OBSERVABILITY STACK
|
||||
|
||||
Recommended:
|
||||
|
||||
- Prometheus → metrics ingestion
|
||||
- Grafana → visualization
|
||||
- Cockpit → control plane
|
||||
- Fester UI → execution DAG + causal graph
|
||||
|
||||
---
|
||||
|
||||
# 🧠 8. GOLDEN RULE
|
||||
|
||||
> Never trust a build you cannot replay.
|
||||
🧭 WHAT YOU NOW HAVE (SYSTEM MATURITY STATE)
|
||||
|
||||
You’ve effectively reached:
|
||||
|
||||
🧠 Level 1 — Distributed Compiler System
|
||||
|
||||
(distcc-like foundation)
|
||||
|
||||
🧠 Level 2 — Smart Scheduler
|
||||
|
||||
(load/thermal/cache aware)
|
||||
|
||||
🧠 Level 3 — Observability System
|
||||
|
||||
(metrics + Grafana + cockpit)
|
||||
|
||||
🧠 Level 4 — Causal Execution Graph
|
||||
|
||||
(why decisions happen)
|
||||
|
||||
🧠 Level 5 — Replayable Build Brain
|
||||
|
||||
(session + timeline + autopsy)
|
||||
|
|
@ -0,0 +1,322 @@
|
|||
# 🧠 Fester
|
||||
|
||||
**A distributed, DAG-driven build execution system with real-time scheduling, thermal/load awareness, cache-aware execution, and deterministic replay/debugging.**
|
||||
|
||||
[](LICENSE)
|
||||
[](https://www.python.org/)
|
||||
[](https://fastapi.tiangolo.com/)
|
||||
|
||||
---
|
||||
|
||||
## 📸 Screenshots
|
||||
|
||||
| Dashboard | Live DAG | Replay |
|
||||
|-----------|----------|--------|
|
||||
|  |  |  |
|
||||
|
||||
| Sessions | Metrics | Cause Graph |
|
||||
|----------|---------|-------------|
|
||||
|  |  |  |
|
||||
|
||||
| Timeline | Debugger |
|
||||
|----------|----------|
|
||||
|  |  |
|
||||
|
||||
---
|
||||
|
||||
## 🎯 What Fester Does
|
||||
|
||||
Fester turns a cluster of machines into a single, observable build brain:
|
||||
|
||||
- **DAG-driven execution** — every build is compiled into a directed acyclic graph of actions, with dependencies honored and parallel branches run concurrently
|
||||
- **Smart scheduler** — picks the best node for each action based on CPU load, temperature, policy constraints, and historical instability
|
||||
- **Real-time observability** — every event (node state, schedule decision, task lifecycle, cache hit/miss, failure) streams live to the UI via WebSocket
|
||||
- **Deterministic replay** — every build session is journaled to SQLite; scrub back through the timeline to see exactly what happened, when, and why
|
||||
- **Failure autopsy** — when an action fails, trace its dependency chain backward, see the last scheduler decision, and compute the forward blast radius
|
||||
- **Cause graph** — a post-hoc reasoning layer that builds a causal graph from system events, so you can ask "why did this node make this decision?"
|
||||
- **Cache layer** — MinIO distributed cache + optional Btrfs CoW reflinks + QCOW2 workspace snapshots
|
||||
|
||||
## 🏗️ Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ Fester Cluster │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │
|
||||
│ │ Backend │◄──►│ EventBus │◄──►│ PipelineEngine │ │
|
||||
│ │ (FastAPI) │ │ (singleton)│ │ (DAG executor) │ │
|
||||
│ └──────┬──────┘ └──────┬──────┘ └──────────┬──────────┘ │
|
||||
│ │ │ │ │
|
||||
│ │ │ ┌───────────────────┼──────────┐ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │ ▼ ▼ ▼ │ │
|
||||
│ ┌──────┴──────┐ ┌─────────────┐ ┌─────────────────┐ │ │
|
||||
│ │ WebSocket │ │ Timeline │ │ Scheduler │ │ │
|
||||
│ │ Stream │ │ Store │ │ (weighted/thermal│ │ │
|
||||
│ │ (/ws, etc) │ │ (SQLite) │ │ /cache-aware) │ │ │
|
||||
│ └──────┬──────┘ └──────┬──────┘ └────────┬────────┘ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ ┌───────┴────────┐ │ │
|
||||
│ │ │ │ Node Registry │ │ │
|
||||
│ │ │ │ (probe + drift)│ │ │
|
||||
│ ▼ ▼ └────────────────┘ │ │
|
||||
│ ┌──────────────────────────────────────────────────────┐ │ │
|
||||
│ │ UI (8 pages) │ │ │
|
||||
│ │ Dashboard │ Live DAG │ Replay │ Sessions │ Metrics │ │ │
|
||||
│ │ Cause Graph │ Timeline │ Debugger │ │ │
|
||||
│ └──────────────────────────────────────────────────────┘ │ │
|
||||
│ │ │
|
||||
│ ┌────────────┐ ┌────────────┐ ┌────────────┐ ┌────────┐│ │
|
||||
│ │ MinIO Cache│ │ Btrfs CAS │ │ QCOW2 Snap │ │ tmux ││ │
|
||||
│ │ (optional) │ │ (optional)│ │ (optional) │ │ runtime││ │
|
||||
│ └────────────┘ └────────────┘ └────────────┘ └────────┘│ │
|
||||
│ │ │
|
||||
└──────────────────────────────────────────────────────────────┘ │
|
||||
│
|
||||
▲ │
|
||||
│ HTTP :8787/status (probe) │
|
||||
│ │
|
||||
┌───────────────┴────────────────────────────────┐ │
|
||||
│ Cluster Nodes │ │
|
||||
│ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐│ │
|
||||
│ │ x99-v3 │ │ x99-v4 │ │ rpi-1 │ │ ryzen-1││ │
|
||||
│ │(x86_64)│ │(x86_64)│ │(arm64) │ │(x86_64)││ │
|
||||
│ └────────┘ └────────┘ └────────┘ └────────┘│ │
|
||||
└────────────────────────────────────────────────┘ │
|
||||
```
|
||||
|
||||
## 📁 Project Layout
|
||||
|
||||
```
|
||||
fester/
|
||||
├── backend/ # Python backend (FastAPI)
|
||||
│ ├── main.py # App entrypoint + 71 routes
|
||||
│ ├── api/ # REST routers + WebSocket hub
|
||||
│ │ ├── build.py # /api/build
|
||||
│ │ ├── nodes.py # /api/nodes (list, policy, probe)
|
||||
│ │ ├── replay.py # /replay/start, /events
|
||||
│ │ ├── autopsy.py # /autopsy/{sid}/{action}
|
||||
│ │ ├── timeline.py # /timeline/{node}, /rewind/{i}
|
||||
│ │ ├── debugger.py # /debugger/{pause,resume,step}
|
||||
│ │ ├── pipeline_control.py # /api/pipeline/{state,retry,...}
|
||||
│ │ ├── metrics.py # /metrics (Prometheus) + /api/metrics/json
|
||||
│ │ └── cause.py # /api/cause/{explain,trace,events}
|
||||
│ ├── pipeline/ # Build execution
|
||||
│ │ ├── engine.py # DAG executor with pause/step/critical-path
|
||||
│ │ └── runner.py # BuildRunner (lifecycle + persistence)
|
||||
│ ├── scheduler/ # Node selection
|
||||
│ │ └── optimizer.py # choose_best_node (weighted scoring)
|
||||
│ ├── analysis/ # Reasoning layer
|
||||
│ │ ├── cause_graph.py # Causal graph from events
|
||||
│ │ ├── timeline_store.py # SQLite-backed event journal
|
||||
│ │ ├── failure_autopsy.py # Backward dependency tracing
|
||||
│ │ └── failure_propagation.py # Forward blast radius
|
||||
│ ├── events/ # Event infrastructure
|
||||
│ │ ├── bus.py # Singleton EventBus
|
||||
│ │ └── schema.py # FesterEvent dataclass + EventType enum
|
||||
│ ├── nodes/ # Node management
|
||||
│ │ ├── probe.py # HTTP agent probing (async + sync)
|
||||
│ │ ├── roles.py # NodeRole dataclass
|
||||
│ │ └── state_model.py # NodeStateRegistry
|
||||
│ ├── storage/ # Storage layer
|
||||
│ │ ├── btrfs_cas.py # CoW reflink snapshots
|
||||
│ │ ├── qcow2_freeze.py # QCOW2 workspace freezing
|
||||
│ │ ├── tmpfs.py # tmpfs workspace acceleration
|
||||
│ │ └── sqlite_db.py # Persistent storage (builds, sessions, events)
|
||||
│ ├── integrations/ # External tool integrations
|
||||
│ │ ├── tmux.py # Detached tmux action runtime
|
||||
│ │ ├── mosh.py # mosh/ssh shell command builder
|
||||
│ │ ├── forgejo.py # Forgejo push webhook handler
|
||||
│ │ ├── lxc.py # LXC container execution
|
||||
│ │ └── libvirt.py # libvirt VM execution
|
||||
│ ├── executor/ # Runtime router (host/lxc/libvirt/tmux)
|
||||
│ ├── cache/ # MinIO distributed cache
|
||||
│ ├── graph/ # DAG construction + critical path
|
||||
│ ├── metrics/ # Prometheus exporter + observability hub
|
||||
│ ├── policy/ # Policy engine (rules + overrides)
|
||||
│ └── targets/ # Build target catalog (Gentoo, Buildroot, ...)
|
||||
├── ui/ # Frontend (vanilla JS, no build step)
|
||||
│ ├── app.js # Shared shell (WS, toasts, topbar, health)
|
||||
│ ├── live_dag.html # Real-time DAG with layered layout
|
||||
│ ├── replay.html # Timeline scrubber + autopsy
|
||||
│ ├── sessions.html # Build history + tmux output viewer
|
||||
│ ├── metrics.html # Live charts + per-node cards
|
||||
│ ├── cause.html # Cause graph + blast radius
|
||||
│ ├── timeline.html # Per-node event drill-down
|
||||
│ └── debugger.html # Step-through execution control
|
||||
├── cli/ # CLI tool (35+ subcommands)
|
||||
│ └── fester.py
|
||||
├── cockpit/ # Cockpit module (optional)
|
||||
├── docs/ # Documentation + screenshots
|
||||
├── scripts/ # Helper scripts (mock agent, syntax checker)
|
||||
├── pyproject.toml # Python packaging
|
||||
├── bootstrap.sh # One-shot dependency installer
|
||||
├── run.sh # Start the backend
|
||||
└── config.yaml # Cluster configuration
|
||||
```
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Clone
|
||||
git clone https://git.dcos.net/dcosnet/fester.git
|
||||
cd fester
|
||||
|
||||
# 2. Bootstrap (creates venv, installs deps, sets up config)
|
||||
./bootstrap.sh
|
||||
|
||||
# 3. Run
|
||||
./run.sh
|
||||
|
||||
# 4. Open the UI
|
||||
open http://localhost:8080
|
||||
```
|
||||
|
||||
See **[quickstart.md](quickstart.md)** for the full 5-minute walkthrough.
|
||||
|
||||
## 🛠️ Prerequisites
|
||||
|
||||
- **Python 3.12+**
|
||||
- **pip** (for dependency installation)
|
||||
- **Optional** (features activate automatically when installed):
|
||||
- `tmux` — for the tmux action runtime (live output viewing)
|
||||
- `mosh` + `ssh` — for the "Shell into node" UI button
|
||||
- `qemu-utils` — for QCOW2 workspace snapshots
|
||||
- `rsync` — for workspace freeze operations
|
||||
- `btrfs-progs` — for CoW reflink snapshots (requires btrfs filesystem)
|
||||
- `minio` server — for distributed cache (otherwise falls back to local)
|
||||
|
||||
## 📊 Tech Stack
|
||||
|
||||
| Layer | Technology |
|
||||
|-------|------------|
|
||||
| Backend | Python 3.12, FastAPI 0.128, uvicorn, pydantic 2 |
|
||||
| Realtime | WebSockets (3 channels: `/ws`, `/ws-targets`, `/ws-debugger`) |
|
||||
| Storage | SQLite (WAL mode) for builds/sessions/events; MinIO for distributed cache |
|
||||
| Frontend | Vanilla HTML/CSS/JS (no build step, no framework, ~12KB total) |
|
||||
| Observability | Prometheus exposition + Grafana dashboard configs |
|
||||
| CLI | Python argparse (35+ subcommands) |
|
||||
| Integrations | tmux, mosh, LXC, libvirt, Forgejo, distcc, ccache |
|
||||
|
||||
## 🎛️ Configuration
|
||||
|
||||
Fester reads from `config.yaml` (or `FESTER_CONFIG` env var) at startup:
|
||||
|
||||
```yaml
|
||||
master:
|
||||
name: fester-master
|
||||
role: control
|
||||
|
||||
nodes:
|
||||
- name: x99-v3
|
||||
host: 192.168.1.10
|
||||
max_jobs: 24
|
||||
- name: x99-v4
|
||||
host: 192.168.1.11
|
||||
max_jobs: 30
|
||||
|
||||
projects:
|
||||
- name: linux-tool
|
||||
repo: https://forgejo.local/linux-tool.git
|
||||
targets:
|
||||
debian: "make clean && make debian"
|
||||
arch: "make clean && make arch"
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Default | Purpose |
|
||||
|----------|---------|---------|
|
||||
| `FESTER_CONFIG` | `config.yaml` | Path to YAML config |
|
||||
| `FESTER_DB_PATH` | `/var/lib/fester/fester.db` (with fallbacks) | SQLite database location |
|
||||
| `FESTER_NO_DRIFT` | unset | Set to `1` to disable synthetic drift when agents unreachable |
|
||||
| `FESTER_AUTOBUILD` | unset | Set to `1` to auto-trigger a build every 60s |
|
||||
| `FESTER_ROLE_DB` | `/etc/fester/node_roles.json` | Node role overrides |
|
||||
| `FESTER_CACHE_DIR` | `/var/lib/fester/cache` | Local cache directory |
|
||||
| `FESTER_STORAGE_CONFIG` | `/etc/fester/storage.json` | Storage layer config |
|
||||
| `FESTER_API` | `http://localhost:8080` | CLI: backend URL |
|
||||
| `FESTER_WS` | `ws://localhost:8080/ws` | CLI: WebSocket URL |
|
||||
|
||||
## 🔌 API Surface
|
||||
|
||||
71 routes total. Key endpoints:
|
||||
|
||||
| Method | Path | Purpose |
|
||||
|--------|------|---------|
|
||||
| `POST` | `/api/build` | Kick off a build |
|
||||
| `GET` | `/api/builds` | Build history |
|
||||
| `POST` | `/api/builds/{id}/cancel` | Cancel a running build |
|
||||
| `GET` | `/api/nodes` | List cluster nodes (with live metrics) |
|
||||
| `POST` | `/api/nodes/{name}/policy` | Set node policy (preferred/avoid/neutral) |
|
||||
| `POST` | `/api/nodes/{name}/probe` | Manually probe a node's agent |
|
||||
| `POST` | `/api/nodes/{name}/shell` | Build mosh/ssh shell command |
|
||||
| `GET` | `/api/metrics/json` | JSON metrics snapshot |
|
||||
| `GET` | `/metrics` | Prometheus exposition |
|
||||
| `GET` | `/api/cause/explain/{node}` | Causal chain for a node |
|
||||
| `GET` | `/api/propagation/{action}` | Blast radius if action fails |
|
||||
| `POST` | `/replay/start` | Start a replay session |
|
||||
| `GET` | `/replay/events` | Full event journal |
|
||||
| `GET` | `/autopsy/{sid}/{action}` | Failure autopsy |
|
||||
| `GET` | `/api/storage/status` | Storage capabilities |
|
||||
| `POST` | `/api/storage/freeze` | Freeze workdir into qcow2 |
|
||||
| `GET` | `/api/actions/active` | List active tmux sessions |
|
||||
| `WS` | `/ws` | Main event stream |
|
||||
| `WS` | `/ws-debugger` | Debugger control channel |
|
||||
| `WS` | `/ws-targets` | Target toggle notifications |
|
||||
|
||||
Full route list: `curl http://localhost:8080/openapi.json | jq '.paths | keys[]'`
|
||||
|
||||
## 🖥️ CLI
|
||||
|
||||
```bash
|
||||
# Build + watch
|
||||
fester build --cmd "make -j$(nproc)" --dir /home/user/linux --watch
|
||||
|
||||
# List builds
|
||||
fester builds
|
||||
|
||||
# Node management
|
||||
fester node list
|
||||
fester node set-policy x99-v3 preferred
|
||||
fester node probe x99-v3
|
||||
|
||||
# Cause analysis
|
||||
fester cause explain x99-v3 build_kernel
|
||||
fester blast build_kernel
|
||||
|
||||
# Live event stream
|
||||
fester stream
|
||||
|
||||
# Health check
|
||||
fester health
|
||||
```
|
||||
|
||||
Run `fester --help` for the full list of 35+ subcommands.
|
||||
|
||||
## 📚 Documentation
|
||||
|
||||
- **[quickstart.md](quickstart.md)** — 5-minute getting started guide
|
||||
- **[CHEATSHEET.md](CHEATSHEET.md)** — Operator survival guide
|
||||
- **[CONTRIBUTING.md](CONTRIBUTING.md)** — How to contribute
|
||||
- **[CHANGELOG.md](CHANGELOG.md)** — Version history
|
||||
- **[docs/screenshots/](docs/screenshots/)** — UI screenshots
|
||||
|
||||
## 🐳 Docker
|
||||
|
||||
```bash
|
||||
docker build -t fester .
|
||||
docker run -p 8080:8080 -v fester-data:/var/lib/fester fester
|
||||
```
|
||||
|
||||
See **[Dockerfile](Dockerfile)** for details.
|
||||
|
||||
## 🔐 Security
|
||||
|
||||
Fester is designed to run behind your existing network controls (NAT, firewalls like OPNsense, etc.). There is **no built-in auth** — assume the network is trusted. If you need to expose it publicly, put it behind a reverse proxy with auth (nginx + OAuth2 Proxy, Traefik + Authelia, etc.).
|
||||
|
||||
## ⚠️ License
|
||||
|
||||
GNU Affero General Public License v3.0 — see **[LICENSE](LICENSE)**.
|
||||
|
||||
Software is provided "AS IS", without warranty of any kind. This is a high-concurrency distributed execution system — review the safety notes in the license before production use.
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
{
|
||||
"title": "Fester Advanced Cluster Intelligence",
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 38,
|
||||
"version": 2,
|
||||
"refresh": "5s",
|
||||
"panels": [
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Node CPU Load",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "fester_node_cpu",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
}
|
||||
},
|
||||
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Node Temperature (Thermal Risk)",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "fester_node_temp",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "celsius",
|
||||
"thresholds": {
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 65 },
|
||||
{ "color": "orange", "value": 75 },
|
||||
{ "color": "red", "value": 80 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Thermal Trend (Prediction Signal)",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "deriv(fester_node_temp[2m])",
|
||||
"legendFormat": "{{node}} temp/s"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 6 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Danger Nodes (>80C)",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(fester_node_temp > 80)"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 12, "y": 8, "w": 6, "h": 6 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Warming Nodes (>70C)",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count(fester_node_temp > 70)"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 18, "y": 8, "w": 6, "h": 6 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Pipeline Throughput",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(fester_pipeline_actions_total[1m])",
|
||||
"legendFormat": "{{state}}"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 0, "y": 14, "w": 12, "h": 7 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Cache Hit Rate Per Target",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(fester_cache_hits_total[5m])",
|
||||
"legendFormat": "{{target}}"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 12, "y": 14, "w": 12, "h": 7 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Cache Efficiency Ratio",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(fester_cache_hits_total[5m]) / rate(fester_pipeline_actions_total[5m])",
|
||||
"legendFormat": "efficiency"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 0, "y": 21, "w": 12, "h": 6 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Scheduler Node Usage",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (node) (fester_scheduler_score)",
|
||||
"legendFormat": "{{node}}"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 12, "y": 21, "w": 12, "h": 6 }
|
||||
},
|
||||
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Scheduler Target Distribution",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (target) (fester_scheduler_score)",
|
||||
"legendFormat": "{{target}}"
|
||||
}
|
||||
],
|
||||
"gridPos": { "x": 0, "y": 27, "w": 24, "h": 6 }
|
||||
}
|
||||
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
scrape_configs:
|
||||
- job_name: 'fester'
|
||||
static_configs:
|
||||
- targets: ['localhost:9109']
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
# analysis/failure_autopsy.py
|
||||
|
||||
class FailureAutopsy:
|
||||
|
||||
def __init__(self, journal, critical_path=None):
|
||||
self.journal = journal
|
||||
self.critical_path = critical_path
|
||||
|
||||
# -----------------------------
|
||||
# FIND FAILURE EVENTS
|
||||
# -----------------------------
|
||||
def find_failures(self):
|
||||
|
||||
return [
|
||||
e for e in self.journal.events
|
||||
if e["type"] == "execution_result"
|
||||
and e["data"].get("state") == "failed"
|
||||
]
|
||||
|
||||
# -----------------------------
|
||||
# TRACE BACKWARD DEPENDENCY CHAIN
|
||||
# -----------------------------
|
||||
def trace_dependencies(self, action_name):
|
||||
|
||||
trace = []
|
||||
visited = set()
|
||||
|
||||
def walk(name):
|
||||
if name in visited:
|
||||
return
|
||||
visited.add(name)
|
||||
|
||||
events = self.journal.trace_action(name)
|
||||
|
||||
trace.append({
|
||||
"action": name,
|
||||
"events": events
|
||||
})
|
||||
|
||||
for e in events:
|
||||
deps = e["data"].get("deps", [])
|
||||
for d in deps:
|
||||
walk(d)
|
||||
|
||||
walk(action_name)
|
||||
|
||||
return trace
|
||||
|
||||
# -----------------------------
|
||||
# GET LAST SCHEDULER DECISION
|
||||
# -----------------------------
|
||||
def last_decision(self, action_name):
|
||||
|
||||
events = self.journal.trace_action(action_name)
|
||||
|
||||
for e in reversed(events):
|
||||
if e["type"] == "schedule_decision":
|
||||
return e["data"]
|
||||
|
||||
return None
|
||||
|
||||
# -----------------------------
|
||||
# FULL AUTOPSY REPORT
|
||||
# -----------------------------
|
||||
def report(self, action_name):
|
||||
|
||||
failures = self.find_failures()
|
||||
|
||||
target_failure = None
|
||||
|
||||
for f in failures:
|
||||
if f["data"].get("action") == action_name:
|
||||
target_failure = f
|
||||
break
|
||||
|
||||
if not target_failure:
|
||||
return {
|
||||
"status": "no_failure_found",
|
||||
"action": action_name
|
||||
}
|
||||
|
||||
deps_trace = self.trace_dependencies(action_name)
|
||||
last_sched = self.last_decision(action_name)
|
||||
|
||||
return {
|
||||
"status": "failure_detected",
|
||||
"action": action_name,
|
||||
|
||||
"failure_event": target_failure,
|
||||
|
||||
"last_scheduler_decision": last_sched,
|
||||
|
||||
"dependency_trace": deps_trace,
|
||||
|
||||
"on_critical_path": (
|
||||
action_name in (self.critical_path or {}).get("score_map", {})
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
# analysis/failure_propagation.py
|
||||
|
||||
class FailurePropagation:
|
||||
|
||||
def __init__(self, graph):
|
||||
"""
|
||||
graph = { node: [deps...] }
|
||||
"""
|
||||
self.graph = graph
|
||||
|
||||
# reverse graph for downstream impact
|
||||
self.reverse = self._build_reverse(graph)
|
||||
|
||||
def _build_reverse(self, graph):
|
||||
reverse = {}
|
||||
|
||||
for node, deps in graph.items():
|
||||
if node not in reverse:
|
||||
reverse[node] = []
|
||||
|
||||
for d in deps:
|
||||
reverse.setdefault(d, []).append(node)
|
||||
|
||||
return reverse
|
||||
|
||||
# -----------------------------
|
||||
# BACKWARD FAILURE TRACE
|
||||
# -----------------------------
|
||||
def propagate_backward(self, failed_node):
|
||||
"""
|
||||
Walk dependencies backward (root cause side)
|
||||
"""
|
||||
visited = set()
|
||||
impacted = []
|
||||
|
||||
def walk(node):
|
||||
for dep in self.graph.get(node, []):
|
||||
if dep in visited:
|
||||
continue
|
||||
visited.add(dep)
|
||||
|
||||
impacted.append({
|
||||
"node": dep,
|
||||
"type": "root_cause_candidate"
|
||||
})
|
||||
|
||||
walk(dep)
|
||||
|
||||
walk(failed_node)
|
||||
|
||||
return impacted
|
||||
|
||||
# -----------------------------
|
||||
# FORWARD IMPACT TRACE
|
||||
# -----------------------------
|
||||
def propagate_forward(self, failed_node):
|
||||
"""
|
||||
Walk dependents forward (blast radius side)
|
||||
"""
|
||||
visited = set()
|
||||
impacted = []
|
||||
|
||||
def walk(node):
|
||||
for child in self.reverse.get(node, []):
|
||||
if child in visited:
|
||||
continue
|
||||
visited.add(child)
|
||||
|
||||
impacted.append({
|
||||
"node": child,
|
||||
"type": "downstream_affected"
|
||||
})
|
||||
|
||||
walk(child)
|
||||
|
||||
walk(failed_node)
|
||||
|
||||
return impacted
|
||||
|
||||
# -----------------------------
|
||||
# FULL FAILURE FIELD MAP
|
||||
# -----------------------------
|
||||
def map_failure(self, failed_node):
|
||||
|
||||
backward = self.propagate_backward(failed_node)
|
||||
forward = self.propagate_forward(failed_node)
|
||||
|
||||
return {
|
||||
"failed_node": failed_node,
|
||||
"root_cause_candidates": backward,
|
||||
"downstream_impact": forward,
|
||||
"severity": len(forward) + len(backward)
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
# analysis/timeline_store.py
|
||||
|
||||
class TimelineStore:
|
||||
|
||||
def __init__(self):
|
||||
self.events = []
|
||||
|
||||
# -----------------------------
|
||||
# APPEND EVENT (IMMUTABLE LOG)
|
||||
# -----------------------------
|
||||
def record(self, event):
|
||||
"""
|
||||
Every state transition in the system must go here.
|
||||
"""
|
||||
self.events.append(event)
|
||||
|
||||
# -----------------------------
|
||||
# GET EVENTS FOR NODE
|
||||
# -----------------------------
|
||||
def get_node_events(self, node_name):
|
||||
|
||||
return [
|
||||
e for e in self.events
|
||||
if e.get("data", {}).get("node") == node_name
|
||||
]
|
||||
|
||||
# -----------------------------
|
||||
# GET TIMELINE SNAPSHOT
|
||||
# -----------------------------
|
||||
def snapshot_at(self, index):
|
||||
|
||||
if index < 0 or index >= len(self.events):
|
||||
return None
|
||||
|
||||
return self.events[index]
|
||||
|
||||
# -----------------------------
|
||||
# FIND LAST STATE BEFORE EVENT
|
||||
# -----------------------------
|
||||
def rewind_to_event(self, node_name, event_index):
|
||||
|
||||
history = []
|
||||
|
||||
for i in range(event_index + 1):
|
||||
e = self.events[i]
|
||||
|
||||
if e.get("data", {}).get("node") == node_name:
|
||||
history.append(e)
|
||||
|
||||
return history
|
||||
|
||||
# -----------------------------
|
||||
# FIND LAST SCHEDULER DECISION
|
||||
# -----------------------------
|
||||
def last_scheduler_state(self, node_name, event_index):
|
||||
|
||||
for i in range(event_index, -1, -1):
|
||||
e = self.events[i]
|
||||
|
||||
if e["type"] == "schedule_decision":
|
||||
if e.get("data", {}).get("node") == node_name:
|
||||
return e["data"]
|
||||
|
||||
return None
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
from collections import defaultdict
|
||||
import time
|
||||
|
||||
|
||||
class CauseGraphEngine:
|
||||
"""
|
||||
Builds a causal graph from system events.
|
||||
|
||||
This is NOT execution logic.
|
||||
It is a post-hoc + live reasoning layer.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# node -> causes
|
||||
self.graph = defaultdict(list)
|
||||
|
||||
# event timeline
|
||||
self.events = []
|
||||
|
||||
# last decision context
|
||||
self.last_context = {}
|
||||
|
||||
# bus reference (set by attach_bus)
|
||||
self.bus = None
|
||||
|
||||
# -------------------------------------------------
|
||||
# BUS BINDING
|
||||
# -------------------------------------------------
|
||||
def attach_bus(self, bus):
|
||||
"""Bind to the event bus. Does NOT subscribe — caller does that."""
|
||||
self.bus = bus
|
||||
|
||||
def emit_debug(self, data):
|
||||
"""Emit a debug trace event back to the bus. Use sparingly to avoid
|
||||
recursion: subscribers of `debug` events MUST NOT call emit_debug."""
|
||||
if self.bus is None:
|
||||
return
|
||||
self.bus.emit(
|
||||
"debug",
|
||||
node=None,
|
||||
action=None,
|
||||
state="trace",
|
||||
meta=data,
|
||||
)
|
||||
|
||||
# -------------------------------------------------
|
||||
# INGEST EVENTS
|
||||
# -------------------------------------------------
|
||||
def ingest(self, event: dict):
|
||||
self.events.append(event)
|
||||
|
||||
etype = event.get("type")
|
||||
node = event.get("node")
|
||||
action = event.get("action")
|
||||
|
||||
# store context for causal linking
|
||||
if etype == "task_update":
|
||||
self._handle_task(event)
|
||||
elif etype == "cache_update":
|
||||
self._handle_cache(event)
|
||||
elif etype == "failure":
|
||||
self._handle_failure(event)
|
||||
elif etype == "node_update":
|
||||
self._handle_node(event)
|
||||
|
||||
# -------------------------------------------------
|
||||
# TASK EXECUTION CAUSALITY
|
||||
# -------------------------------------------------
|
||||
def _handle_task(self, event):
|
||||
node = event.get("node")
|
||||
action = event.get("action")
|
||||
state = event.get("state")
|
||||
|
||||
context = {
|
||||
"time": time.time(),
|
||||
"node": node,
|
||||
"action": action,
|
||||
"state": state,
|
||||
"reason": "scheduler_assignment",
|
||||
}
|
||||
|
||||
self.last_context[(node, action)] = context
|
||||
|
||||
self.graph[(node, action)].append(context)
|
||||
|
||||
# -------------------------------------------------
|
||||
# CACHE CAUSALITY
|
||||
# -------------------------------------------------
|
||||
def _handle_cache(self, event):
|
||||
node = event.get("node")
|
||||
action = event.get("action")
|
||||
|
||||
context = {
|
||||
"time": time.time(),
|
||||
"node": node,
|
||||
"action": action,
|
||||
"reason": "cache_hit_or_miss",
|
||||
}
|
||||
|
||||
self.graph[(node, action)].append(context)
|
||||
|
||||
# -------------------------------------------------
|
||||
# FAILURE CAUSALITY
|
||||
# -------------------------------------------------
|
||||
def _handle_failure(self, event):
|
||||
node = event.get("node")
|
||||
action = event.get("action")
|
||||
|
||||
prev = self.last_context.get((node, action), {})
|
||||
|
||||
context = {
|
||||
"time": time.time(),
|
||||
"node": node,
|
||||
"action": action,
|
||||
"reason": "execution_failure",
|
||||
"parent_context": prev,
|
||||
}
|
||||
|
||||
self.graph[(node, action)].append(context)
|
||||
|
||||
# -------------------------------------------------
|
||||
# NODE STATE CAUSALITY
|
||||
# -------------------------------------------------
|
||||
def _handle_node(self, event):
|
||||
node = event.get("node")
|
||||
|
||||
context = {
|
||||
"time": time.time(),
|
||||
"node": node,
|
||||
"reason": "node_state_update",
|
||||
"state": event.get("node_state", {}),
|
||||
}
|
||||
|
||||
self.graph[(node, None)].append(context)
|
||||
|
||||
# -------------------------------------------------
|
||||
# QUERY INTERFACE
|
||||
# -------------------------------------------------
|
||||
def explain(self, node, action=None):
|
||||
"""
|
||||
Returns causal chain for a given execution unit.
|
||||
|
||||
If action is None, returns ALL events for the given node
|
||||
(any action). Otherwise returns events for the specific
|
||||
(node, action) pair.
|
||||
"""
|
||||
if action is not None:
|
||||
return self.graph.get((node, action), [])
|
||||
|
||||
# Node-only query: scan all keys starting with this node
|
||||
results = []
|
||||
for (n, a), events in self.graph.items():
|
||||
if n == node:
|
||||
results.extend(events)
|
||||
# Sort by time
|
||||
results.sort(key=lambda e: e.get("time", 0))
|
||||
return results
|
||||
|
||||
def full_trace(self):
|
||||
return dict(self.graph)
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
"""
|
||||
Failure Autopsy — post-hoc root cause analysis for a failed action.
|
||||
|
||||
Given a journal (TimelineStore) and an action name, produces:
|
||||
- the failure event itself
|
||||
- the backward dependency trace (recursive walk)
|
||||
- the last scheduler decision for that action
|
||||
- whether the action is on the critical path
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
class FailureAutopsy:
|
||||
def __init__(self, journal, critical_path: Optional[Dict[str, Any]] = None):
|
||||
"""
|
||||
journal: TimelineStore (or anything with `trace_action` + `find_failures`)
|
||||
critical_path: optional dict with at least `score_map` (set of action names)
|
||||
"""
|
||||
self.journal = journal
|
||||
self.critical_path = critical_path or {}
|
||||
|
||||
# -----------------------------
|
||||
# FIND FAILURE EVENTS
|
||||
# -----------------------------
|
||||
def find_failures(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
e for e in self.journal.find_failures()
|
||||
if (e.get("data", {}).get("state") == "failed" or e.get("state") == "failed")
|
||||
]
|
||||
|
||||
# -----------------------------
|
||||
# TRACE BACKWARD DEPENDENCY CHAIN
|
||||
# -----------------------------
|
||||
def trace_dependencies(self, action_name: str) -> List[Dict[str, Any]]:
|
||||
trace: List[Dict[str, Any]] = []
|
||||
visited: set = set()
|
||||
|
||||
def walk(name: str):
|
||||
if name in visited:
|
||||
return
|
||||
visited.add(name)
|
||||
|
||||
events = self.journal.trace_action(name)
|
||||
trace.append({"action": name, "events": events})
|
||||
|
||||
for e in events:
|
||||
d = e.get("data", e)
|
||||
deps = d.get("deps", [])
|
||||
for dep in deps:
|
||||
walk(dep)
|
||||
|
||||
walk(action_name)
|
||||
return trace
|
||||
|
||||
# -----------------------------
|
||||
# GET LAST SCHEDULER DECISION
|
||||
# -----------------------------
|
||||
def last_decision(self, action_name: str) -> Optional[Dict[str, Any]]:
|
||||
events = self.journal.trace_action(action_name)
|
||||
for e in reversed(events):
|
||||
if e.get("type") == "schedule_decision":
|
||||
return e.get("data", e)
|
||||
# Fallback: return last task_update with a node assignment
|
||||
for e in reversed(events):
|
||||
d = e.get("data", e)
|
||||
if d.get("node"):
|
||||
return {
|
||||
"node": d.get("node"),
|
||||
"score": d.get("score"),
|
||||
"reason": "scheduler_assignment",
|
||||
}
|
||||
return None
|
||||
|
||||
# -----------------------------
|
||||
# FULL AUTOPSY REPORT
|
||||
# -----------------------------
|
||||
def report(self, action_name: str) -> Dict[str, Any]:
|
||||
failures = self.find_failures()
|
||||
|
||||
target_failure: Optional[Dict[str, Any]] = None
|
||||
for f in failures:
|
||||
d = f.get("data", f)
|
||||
if d.get("action") == action_name:
|
||||
target_failure = f
|
||||
break
|
||||
|
||||
if not target_failure:
|
||||
return {
|
||||
"status": "no_failure_found",
|
||||
"action": action_name,
|
||||
}
|
||||
|
||||
deps_trace = self.trace_dependencies(action_name)
|
||||
last_sched = self.last_decision(action_name)
|
||||
|
||||
score_map = self.critical_path.get("score_map", {}) if isinstance(self.critical_path, dict) else {}
|
||||
|
||||
return {
|
||||
"status": "failure_detected",
|
||||
"action": action_name,
|
||||
"failure_event": target_failure,
|
||||
"last_scheduler_decision": last_sched,
|
||||
"dependency_trace": deps_trace,
|
||||
"on_critical_path": action_name in score_map,
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
"""
|
||||
Failure Propagation — given a failed action, compute the forward blast radius.
|
||||
|
||||
The "blast radius" is the set of downstream actions that depend (transitively)
|
||||
on the failed action and therefore cannot complete.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Set
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
class FailurePropagator:
|
||||
"""
|
||||
Maintains a reverse dependency index: for each action, the set of
|
||||
actions that depend on it. Given a failed action, walks the reverse
|
||||
graph to compute the impact set.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# action -> set of actions that depend on it
|
||||
self._revdeps: Dict[str, Set[str]] = defaultdict(set)
|
||||
# action -> direct deps (so we can rebuild revdeps if needed)
|
||||
self._deps: Dict[str, List[str]] = {}
|
||||
|
||||
# -----------------------------
|
||||
# INGEST
|
||||
# -----------------------------
|
||||
def register_action(self, action: str, deps: List[str]):
|
||||
"""Register an action and its direct dependencies."""
|
||||
self._deps[action] = list(deps)
|
||||
for d in deps:
|
||||
self._revdeps[d].add(action)
|
||||
|
||||
def ingest_event(self, event: Dict[str, Any]):
|
||||
"""Hook for EventBus subscription — picks up pipeline events
|
||||
that declare dependencies and indexes them.
|
||||
|
||||
Handles both flat FesterEvent shape (deps in `meta.deps`) and
|
||||
legacy nested shape (deps in `data.deps`).
|
||||
"""
|
||||
# Try flat shape first: action + deps at top level or in meta
|
||||
action = event.get("action")
|
||||
deps = event.get("deps")
|
||||
if deps is None and isinstance(event.get("meta"), dict):
|
||||
deps = event["meta"].get("deps")
|
||||
# Legacy nested shape
|
||||
if action is None or deps is None:
|
||||
data = event.get("data")
|
||||
if isinstance(data, dict):
|
||||
if action is None:
|
||||
action = data.get("action")
|
||||
if deps is None:
|
||||
deps = data.get("deps")
|
||||
if action and deps:
|
||||
self.register_action(action, deps)
|
||||
|
||||
# -----------------------------
|
||||
# QUERY
|
||||
# -----------------------------
|
||||
def downstream(self, action: str) -> List[str]:
|
||||
"""Return the transitive set of actions that depend on `action`."""
|
||||
seen: Set[str] = set()
|
||||
stack = [action]
|
||||
while stack:
|
||||
cur = stack.pop()
|
||||
for child in self._revdeps.get(cur, set()):
|
||||
if child not in seen:
|
||||
seen.add(child)
|
||||
stack.append(child)
|
||||
return sorted(seen)
|
||||
|
||||
def impact_report(self, failed_action: str) -> Dict[str, Any]:
|
||||
downstream = self.downstream(failed_action)
|
||||
return {
|
||||
"failed_action": failed_action,
|
||||
"direct_dependents": sorted(self._revdeps.get(failed_action, set())),
|
||||
"transitive_impact": downstream,
|
||||
"impact_count": len(downstream),
|
||||
"total_actions_known": len(self._deps),
|
||||
}
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
"""
|
||||
Timeline Store — immutable event log indexed by node + by absolute index.
|
||||
|
||||
Now backed by SQLite (via backend.storage.sqlite_db) with an in-memory
|
||||
cache for fast recent reads. Falls back to pure in-memory if storage
|
||||
is unavailable.
|
||||
|
||||
Used by:
|
||||
- backend/api/timeline.py (REST endpoints)
|
||||
- backend/analysis/failure_autopsy.py (dependency tracing)
|
||||
- replay UI (via /replay/events)
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from typing import Any, Dict, List, Optional
|
||||
import threading
|
||||
import time
|
||||
|
||||
# Optional storage backend
|
||||
try:
|
||||
from backend.storage.sqlite_db import STORAGE
|
||||
except Exception:
|
||||
STORAGE = None
|
||||
|
||||
|
||||
class TimelineStore:
|
||||
"""
|
||||
Append-only event journal with:
|
||||
- per-node indices for fast node lookups
|
||||
- per-action indices for fast action tracing
|
||||
- snapshot-at-index for replay rewinding
|
||||
- SQLite persistence (with in-memory cache for recent events)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._events: List[Dict[str, Any]] = []
|
||||
self._by_node: Dict[str, List[int]] = defaultdict(list)
|
||||
self._by_action: Dict[str, List[int]] = defaultdict(list)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# On startup, load recent events from storage
|
||||
if STORAGE:
|
||||
self._load_from_storage()
|
||||
|
||||
def _load_from_storage(self, limit: int = 10000):
|
||||
"""Load recent events from SQLite into the in-memory cache."""
|
||||
try:
|
||||
events = STORAGE.get_events(limit=limit)
|
||||
for e in events:
|
||||
self._index_event(e)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _index_event(self, event: Dict[str, Any]):
|
||||
"""Add an event to in-memory indices (no lock — caller must hold it)."""
|
||||
idx = len(self._events)
|
||||
self._events.append(event)
|
||||
node = event.get("node") or (event.get("data", {}) or {}).get("node")
|
||||
if node:
|
||||
self._by_node[node].append(idx)
|
||||
action = event.get("action") or (event.get("data", {}) or {}).get("action")
|
||||
if action:
|
||||
self._by_action[action].append(idx)
|
||||
|
||||
# -----------------------------
|
||||
# APPEND
|
||||
# -----------------------------
|
||||
def append(self, event: Dict[str, Any]) -> int:
|
||||
"""Append an event to the journal. Returns the new index."""
|
||||
with self._lock:
|
||||
if "timestamp" not in event:
|
||||
event["timestamp"] = time.time()
|
||||
self._index_event(event)
|
||||
|
||||
# Persist to SQLite (outside the lock)
|
||||
if STORAGE:
|
||||
try:
|
||||
STORAGE.append_event(event)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
with self._lock:
|
||||
return len(self._events) - 1
|
||||
|
||||
# -----------------------------
|
||||
# READ
|
||||
# -----------------------------
|
||||
def all(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return list(self._events)
|
||||
|
||||
def get_node_events(self, node: str) -> List[Dict[str, Any]]:
|
||||
# Try in-memory first
|
||||
with self._lock:
|
||||
if node in self._by_node:
|
||||
return [self._events[i] for i in self._by_node[node] if i < len(self._events)]
|
||||
# Fall back to storage (queries older events not in cache)
|
||||
if STORAGE:
|
||||
try:
|
||||
return STORAGE.get_node_events(node)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
def get_action_events(self, action: str) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
if action in self._by_action:
|
||||
return [self._events[i] for i in self._by_action[action] if i < len(self._events)]
|
||||
if STORAGE:
|
||||
try:
|
||||
return STORAGE.get_action_events(action)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
def trace_action(self, action: str) -> List[Dict[str, Any]]:
|
||||
"""Alias for get_action_events — kept for autopsy compatibility."""
|
||||
return self.get_action_events(action)
|
||||
|
||||
def find_failures(self) -> List[Dict[str, Any]]:
|
||||
# In-memory
|
||||
with self._lock:
|
||||
in_mem = [
|
||||
e for e in self._events
|
||||
if e.get("type") in ("failure", "task_update")
|
||||
and (e.get("state") == "failed" or e.get("data", {}).get("state") == "failed")
|
||||
]
|
||||
# Storage (may have more)
|
||||
if STORAGE:
|
||||
try:
|
||||
stored = STORAGE.find_failures()
|
||||
# Merge + dedupe by timestamp+action
|
||||
seen = {(e.get("timestamp"), e.get("action")) for e in in_mem}
|
||||
for e in stored:
|
||||
key = (e.get("timestamp"), e.get("action"))
|
||||
if key not in seen:
|
||||
in_mem.append(e)
|
||||
seen.add(key)
|
||||
except Exception:
|
||||
pass
|
||||
return in_mem
|
||||
|
||||
# -----------------------------
|
||||
# REWIND / SNAPSHOT
|
||||
# -----------------------------
|
||||
def snapshot_at(self, index: int) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
safe = max(0, min(index, len(self._events) - 1))
|
||||
events = self._events[:safe + 1]
|
||||
|
||||
# rebuild node + action views
|
||||
nodes: Dict[str, Dict[str, Any]] = {}
|
||||
actions: Dict[str, Dict[str, Any]] = {}
|
||||
for e in events:
|
||||
d = e.get("data", e)
|
||||
n = d.get("node") or e.get("node")
|
||||
a = d.get("action") or e.get("action")
|
||||
s = d.get("state") or e.get("state")
|
||||
if n:
|
||||
nodes[n] = {"name": n, "state": s or "online", "last_event": e.get("timestamp")}
|
||||
if a:
|
||||
actions[a] = {"name": a, "state": s or "pending", "node": n, "last_event": e.get("timestamp")}
|
||||
|
||||
return {
|
||||
"index": safe,
|
||||
"event_count": len(events),
|
||||
"nodes": list(nodes.values()),
|
||||
"actions": list(actions.values()),
|
||||
}
|
||||
|
||||
# -----------------------------
|
||||
# STATS
|
||||
# -----------------------------
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
total = len(self._events)
|
||||
failures = len([e for e in self._events if e.get("type") == "failure"])
|
||||
# If we have storage, get the authoritative count
|
||||
if STORAGE:
|
||||
try:
|
||||
total = STORAGE.event_count()
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"total_events": total,
|
||||
"nodes_tracked": len(self._by_node),
|
||||
"actions_tracked": len(self._by_action),
|
||||
"failures": failures,
|
||||
}
|
||||
|
||||
def reset(self):
|
||||
with self._lock:
|
||||
self._events.clear()
|
||||
self._by_node.clear()
|
||||
self._by_action.clear()
|
||||
|
||||
|
||||
# Process-wide singleton (used by api/timeline.py)
|
||||
STORE = TimelineStore()
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
"""
|
||||
Fester backend — core API facade.
|
||||
|
||||
This module exposes a single `bus`, `registry`, `observability`, `ws_stream`,
|
||||
plus thin wrapper functions for the API layer. It used to be broken
|
||||
(referenced `bus` before creation, imported nonexistent `_endpoint` symbols,
|
||||
duplicated CauseGraphEngine setup). It now wires everything cleanly.
|
||||
"""
|
||||
|
||||
from backend.events.bus import EventBus
|
||||
from backend.metrics.observability import ObservabilityHub
|
||||
from backend.metrics.node_state import NodeStateRegistry
|
||||
from backend.analysis.cause_graph import CauseGraphEngine
|
||||
|
||||
# =========================================================
|
||||
# GLOBAL CORE SYSTEM INITIALIZATION (SINGLETON LAYER)
|
||||
# =========================================================
|
||||
|
||||
bus = EventBus()
|
||||
registry = NodeStateRegistry()
|
||||
observability = ObservabilityHub(registry, bus)
|
||||
observability.attach_bus()
|
||||
|
||||
# Cause graph — subscribes ONCE, no recursion (ingest only)
|
||||
cause_graph = CauseGraphEngine()
|
||||
cause_graph.attach_bus(bus)
|
||||
bus.subscribe(cause_graph.ingest)
|
||||
|
||||
# WebSocket stream — attached lazily by main.py when FastAPI starts,
|
||||
# because the WS layer needs the running app's connection registry.
|
||||
ws_stream = None # set by backend.main when app boots
|
||||
|
||||
|
||||
def attach_ws_stream(stream):
|
||||
"""Called by backend.main during startup to bind a WebSocketStream."""
|
||||
global ws_stream
|
||||
ws_stream = stream
|
||||
ws_stream.attach_bus(bus)
|
||||
|
||||
|
||||
# =========================================================
|
||||
# API REGISTRATION LAYER
|
||||
# Thin wrappers so Cockpit / external callers can bind cleanly.
|
||||
# =========================================================
|
||||
|
||||
# These imports are deliberately placed AFTER the singletons above are
|
||||
# initialized, because some api/* modules may want to import `bus` etc.
|
||||
from backend.api import build as build_api
|
||||
from backend.api import nodes as nodes_api
|
||||
from backend.api import replay as replay_api
|
||||
from backend.api import debugger as debugger_api
|
||||
from backend.api import timeline as timeline_api
|
||||
from backend.api import metrics as metrics_api
|
||||
from backend.api import autopsy as autopsy_api
|
||||
from backend.api import pipeline_control as pipeline_control_api
|
||||
from backend.api import cause as cause_api
|
||||
|
||||
|
||||
def build(spec, nodes, cluster=None, intelligence=None):
|
||||
return build_api.build_endpoint(spec, nodes, cluster, intelligence)
|
||||
|
||||
|
||||
def replay(session_id):
|
||||
return replay_api.replay_endpoint(session_id, registry)
|
||||
|
||||
|
||||
def debugger(session_id):
|
||||
return debugger_api.debugger_endpoint(session_id, registry, bus)
|
||||
|
||||
|
||||
def timeline(session_id):
|
||||
return timeline_api.timeline_endpoint(session_id)
|
||||
|
||||
|
||||
def nodes():
|
||||
return nodes_api.nodes_endpoint(registry)
|
||||
|
||||
|
||||
def metrics():
|
||||
return metrics_api.metrics_endpoint(registry)
|
||||
|
||||
|
||||
def autopsy(session_id):
|
||||
return autopsy_api.autopsy_endpoint(session_id)
|
||||
|
||||
|
||||
def pipeline_control(action, payload=None):
|
||||
return pipeline_control_api.pipeline_control_endpoint(action, payload, bus)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"bus",
|
||||
"registry",
|
||||
"observability",
|
||||
"ws_stream",
|
||||
"cause_graph",
|
||||
"attach_ws_stream",
|
||||
|
||||
"build",
|
||||
"replay",
|
||||
"debugger",
|
||||
"timeline",
|
||||
"nodes",
|
||||
"metrics",
|
||||
"autopsy",
|
||||
"pipeline_control",
|
||||
]
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
"""
|
||||
Autopsy API — post-hoc root cause analysis for failed actions.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from backend.analysis.failure_autopsy import FailureAutopsy
|
||||
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||
from backend.api.replay import SESSION_DB
|
||||
|
||||
router = APIRouter(prefix="/autopsy", tags=["autopsy"])
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# RUN AUTOPSY
|
||||
# -----------------------------
|
||||
@router.get("/{session_id}/{action}")
|
||||
def run_autopsy(session_id: str, action: str):
|
||||
"""Run a failure autopsy for `action` against the replay session `session_id`."""
|
||||
|
||||
# Try to fetch events for this session from SessionDB
|
||||
events = SESSION_DB.get_replay_stream(session_id)
|
||||
|
||||
if not events:
|
||||
# Fallback: use the live timeline store
|
||||
journal = TIMELINE_STORE
|
||||
else:
|
||||
# Wrap the session's event list in a tiny shim that quacks like
|
||||
# the TimelineStore API the FailureAutopsy expects.
|
||||
journal = _JournalShim(events)
|
||||
|
||||
autopsy = FailureAutopsy(journal)
|
||||
return autopsy.report(action)
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Legacy wrapper for api.py facade
|
||||
# -----------------------------
|
||||
def autopsy_endpoint(session_id, action=None):
|
||||
"""Legacy entrypoint — caller passed only session_id; we look up
|
||||
failures across all known sessions."""
|
||||
events = SESSION_DB.get_replay_stream(session_id)
|
||||
if not events:
|
||||
return {"error": "session not found"}
|
||||
journal = _JournalShim(events)
|
||||
autopsy = FailureAutopsy(journal)
|
||||
# If no action given, report on the first failure we find
|
||||
if action is None:
|
||||
failures = autopsy.find_failures()
|
||||
if not failures:
|
||||
return {"status": "no_failures_in_session", "session_id": session_id}
|
||||
action = failures[0].get("data", failures[0]).get("action")
|
||||
if not action:
|
||||
return {"status": "no_actionable_failure", "session_id": session_id}
|
||||
return autopsy.report(action)
|
||||
|
||||
|
||||
class _JournalShim:
|
||||
"""Minimal adapter that lets FailureAutopsy work against a plain list
|
||||
of events (from a replay session) instead of a full TimelineStore."""
|
||||
|
||||
def __init__(self, events):
|
||||
self.events = events
|
||||
|
||||
def find_failures(self):
|
||||
return [
|
||||
e for e in self.events
|
||||
if e.get("type") in ("failure", "task_update", "pipeline_update")
|
||||
and (e.get("state") == "failed" or e.get("data", {}).get("state") == "failed")
|
||||
]
|
||||
|
||||
def trace_action(self, action_name):
|
||||
return [
|
||||
e for e in self.events
|
||||
if (e.get("action") == action_name) or (e.get("data", {}).get("action") == action_name)
|
||||
]
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
"""
|
||||
Build endpoint — kicks off a pipeline run for a BuildSpec.
|
||||
|
||||
NOTE: This is the thin API wrapper. The real scheduling + execution
|
||||
lives in backend/pipeline/engine.py and backend/scheduler/.
|
||||
"""
|
||||
|
||||
from backend.api.build_spec import BuildSpec
|
||||
from backend.pipeline.engine import PipelineEngine
|
||||
from backend.graph.build_graph import BuildGraph
|
||||
from backend.api.api import bus, registry # singletons
|
||||
|
||||
|
||||
async def build_endpoint(spec, nodes, cluster=None, intelligence=None):
|
||||
"""
|
||||
Build a project.
|
||||
|
||||
`spec` may be:
|
||||
- a BuildSpec instance
|
||||
- a dict (will be coerced via BuildSpec.from_request)
|
||||
- a raw dict matching the legacy BuildGraph constructor (with `target`/`output_mode`)
|
||||
|
||||
`nodes`, `cluster`, `intelligence` are accepted for backward-compat but
|
||||
only `nodes` is actually used (the engine has its own node registry).
|
||||
"""
|
||||
# Coerce spec into something BuildGraph can consume.
|
||||
# BuildGraph reads spec.arch, spec.target, spec.toolchain, spec.output_mode.
|
||||
# BuildSpec (the dataclass) doesn't have target/output_mode, so we adapt.
|
||||
if isinstance(spec, BuildSpec):
|
||||
bs = spec
|
||||
elif isinstance(spec, dict):
|
||||
# If caller passed a raw dict with `target`/`output_mode`, wrap it
|
||||
# in a tiny shim object so BuildGraph's attribute access works.
|
||||
if "target" in spec or "output_mode" in spec:
|
||||
bs = _ShimSpec(spec)
|
||||
else:
|
||||
try:
|
||||
bs = BuildSpec.from_request(spec)
|
||||
except KeyError:
|
||||
bs = _ShimSpec(spec)
|
||||
else:
|
||||
# Already a shim or has the attributes BuildGraph expects
|
||||
bs = spec
|
||||
|
||||
# Generate the DAG
|
||||
try:
|
||||
graph = BuildGraph(bs).generate()
|
||||
except Exception as e:
|
||||
# In production we'd log this; for now bubble up a structured error
|
||||
return {"status": "error", "error": f"build_graph_failed: {e}"}
|
||||
|
||||
# Run the pipeline
|
||||
engine = PipelineEngine(nodes, registry, bus)
|
||||
try:
|
||||
results = await engine.run(bs)
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": f"pipeline_failed: {e}"}
|
||||
|
||||
return {
|
||||
"status": "complete",
|
||||
"actions": [{"name": n, "state": s} for n, s in results],
|
||||
"graph_size": len(graph),
|
||||
}
|
||||
|
||||
|
||||
class _ShimSpec:
|
||||
"""Adapter that exposes attribute access over a dict, so BuildGraph
|
||||
can read spec.target / spec.output_mode / spec.arch / spec.toolchain
|
||||
regardless of which shape the caller supplied."""
|
||||
|
||||
def __init__(self, d: dict):
|
||||
self._d = d
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name in self._d:
|
||||
return self._d[name]
|
||||
# Sensible defaults for fields BuildGraph reads
|
||||
defaults = {
|
||||
"arch": "x86_64",
|
||||
"target": "linux-gnu",
|
||||
"toolchain": "gcc",
|
||||
"output_mode": "tar",
|
||||
"project": "unknown",
|
||||
}
|
||||
return defaults.get(name)
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildSpec:
|
||||
|
||||
project: str
|
||||
arch: str
|
||||
distro_target: str
|
||||
toolchain: str
|
||||
|
||||
# -----------------------------
|
||||
# EXECUTION SELECTION
|
||||
# -----------------------------
|
||||
execution: str # distcc | lxc | libvirt | tmux
|
||||
container_backend: str # optional fallback layer
|
||||
|
||||
# -----------------------------
|
||||
# POLICY PROFILE
|
||||
# -----------------------------
|
||||
profile: str = "balanced"
|
||||
|
||||
|
||||
def from_request(req):
|
||||
|
||||
return BuildSpec(
|
||||
project=req["project"],
|
||||
arch=req["arch"],
|
||||
distro_target=req["distro_target"],
|
||||
toolchain=req["toolchain"],
|
||||
execution=req.get("execution", "distcc"),
|
||||
container_backend=req.get("container_backend", "lxc"),
|
||||
profile=req.get("profile", "balanced")
|
||||
)
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
"""
|
||||
Cause graph API — explain why a node/action made its decisions.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from typing import Optional
|
||||
|
||||
from backend.api.api import cause_graph
|
||||
|
||||
router = APIRouter(prefix="/api/cause", tags=["cause"])
|
||||
|
||||
|
||||
@router.get("/explain/{node}")
|
||||
def explain_node(node: str, action: Optional[str] = None):
|
||||
"""Explain the causal chain for a given node (+ optional action)."""
|
||||
chain = cause_graph.explain(node, action)
|
||||
return {
|
||||
"node": node,
|
||||
"action": action,
|
||||
"events": len(chain),
|
||||
"chain": chain,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/trace")
|
||||
def full_trace():
|
||||
"""Return the entire cause graph as a dict."""
|
||||
return {
|
||||
"nodes_tracked": len(cause_graph.graph),
|
||||
"events_seen": len(cause_graph.events),
|
||||
"trace": {str(k): v for k, v in cause_graph.full_trace().items()},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/events")
|
||||
def recent_events(limit: int = 100):
|
||||
"""Return the most recent N ingested events."""
|
||||
events = cause_graph.events[-limit:]
|
||||
return {
|
||||
"count": len(events),
|
||||
"events": events,
|
||||
}
|
||||
|
||||
|
||||
# Legacy wrappers (kept for api.py facade)
|
||||
def explain_node_legacy(cause_graph_instance, node, action=None):
|
||||
return cause_graph_instance.explain(node, action)
|
||||
|
||||
|
||||
def full_cause_trace(cause_graph_instance):
|
||||
return cause_graph_instance.full_trace()
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
"""
|
||||
Debugger API — pause / resume / step an in-flight pipeline.
|
||||
|
||||
The engine is set by main.py on startup via `set_engine(...)`. If no
|
||||
engine is registered (e.g. when running the mock server), endpoints
|
||||
return a structured error rather than crashing.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from typing import Optional, Any
|
||||
|
||||
router = APIRouter(prefix="/debugger", tags=["debugger"])
|
||||
|
||||
# Single in-memory engine reference (set by main.py)
|
||||
_ENGINE: Optional[Any] = None
|
||||
_DEBUG_STATE: dict = {
|
||||
"paused": False,
|
||||
"current_step": 0,
|
||||
"history": [],
|
||||
}
|
||||
|
||||
|
||||
def set_engine(engine):
|
||||
global _ENGINE
|
||||
_ENGINE = engine
|
||||
|
||||
|
||||
def get_engine():
|
||||
return _ENGINE
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# RESUME
|
||||
# -----------------------------
|
||||
@router.post("/resume")
|
||||
def resume():
|
||||
_DEBUG_STATE["paused"] = False
|
||||
if _ENGINE and hasattr(_ENGINE, "resume"):
|
||||
_ENGINE.resume()
|
||||
return {"state": "running"}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# PAUSE
|
||||
# -----------------------------
|
||||
@router.post("/pause")
|
||||
def pause():
|
||||
_DEBUG_STATE["paused"] = True
|
||||
if _ENGINE and hasattr(_ENGINE, "pause"):
|
||||
_ENGINE.pause()
|
||||
return {"state": "paused"}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# STEP
|
||||
# -----------------------------
|
||||
@router.post("/step")
|
||||
def step():
|
||||
if _ENGINE and hasattr(_ENGINE, "step"):
|
||||
_ENGINE.step()
|
||||
_DEBUG_STATE["current_step"] += 1
|
||||
return {"state": "stepped", "step": _DEBUG_STATE["current_step"]}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# STEP BACK (replay-only)
|
||||
# -----------------------------
|
||||
@router.post("/step-back")
|
||||
def step_back():
|
||||
if _DEBUG_STATE["current_step"] > 0:
|
||||
_DEBUG_STATE["current_step"] -= 1
|
||||
return {"state": "stepped_back", "step": _DEBUG_STATE["current_step"]}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# STATE
|
||||
# -----------------------------
|
||||
@router.get("/state")
|
||||
def state():
|
||||
eng = _ENGINE
|
||||
active = eng and hasattr(eng, "is_active") and eng.is_active()
|
||||
return {
|
||||
"paused": _DEBUG_STATE["paused"],
|
||||
"current_step": _DEBUG_STATE["current_step"],
|
||||
"engine_attached": eng is not None,
|
||||
"engine_active": active,
|
||||
}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Legacy wrapper for api.py facade
|
||||
# -----------------------------
|
||||
def debugger_endpoint(session_id, registry=None, bus=None):
|
||||
"""Legacy — return current debugger state for a session."""
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"paused": _DEBUG_STATE["paused"],
|
||||
"current_step": _DEBUG_STATE["current_step"],
|
||||
"engine_attached": _ENGINE is not None,
|
||||
}
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
"""
|
||||
Metrics endpoint — Prometheus exposition + JSON view for the UI.
|
||||
|
||||
Emits metric names matching add-to-grafana-config.json:
|
||||
- fester_node_cpu (gauge, per node)
|
||||
- fester_node_temp (gauge, per node)
|
||||
- fester_node_active_jobs (gauge, per node)
|
||||
- fester_node_instability (gauge, per node)
|
||||
- fester_nodes_total (gauge)
|
||||
- fester_nodes_online (gauge)
|
||||
- fester_cluster_avg_heat (gauge)
|
||||
- fester_cluster_active_jobs (gauge)
|
||||
- fester_pipeline_actions_total (counter, per state) ← grafana expects this
|
||||
- fester_cache_hits_total (counter, per target) ← grafana expects this
|
||||
- fester_scheduler_score (gauge, per node+target)
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from typing import Optional
|
||||
import threading
|
||||
|
||||
from backend.metrics.observability import ObservabilityHub
|
||||
from backend.metrics.node_state import NodeStateRegistry
|
||||
|
||||
router = APIRouter(tags=["metrics"])
|
||||
|
||||
# Process-wide counters (incremented by main.py's bus subscriber)
|
||||
_pipeline_action_counts: dict = {} # state -> count
|
||||
_cache_hit_counts: dict = {} # target -> count
|
||||
_scheduler_scores: dict = {} # (node, target) -> score
|
||||
_metrics_lock = threading.Lock()
|
||||
|
||||
|
||||
def record_pipeline_action(state: str):
|
||||
"""Called by main.py's bus subscriber when a task_update event fires."""
|
||||
with _metrics_lock:
|
||||
_pipeline_action_counts[state] = _pipeline_action_counts.get(state, 0) + 1
|
||||
|
||||
|
||||
def record_cache_hit(target: str):
|
||||
with _metrics_lock:
|
||||
_cache_hit_counts[target] = _cache_hit_counts.get(target, 0) + 1
|
||||
|
||||
|
||||
def record_scheduler_score(node: str, target: str, score: float):
|
||||
with _metrics_lock:
|
||||
_scheduler_scores[(node, target)] = score
|
||||
|
||||
|
||||
def _gather_metrics(registry: NodeStateRegistry):
|
||||
"""Build a JSON-serializable metrics snapshot from the node state registry."""
|
||||
nodes = registry.all()
|
||||
total_jobs = sum(n.active_jobs for n in nodes)
|
||||
avg_heat = (sum(n.temp for n in nodes) / len(nodes)) if nodes else 0.0
|
||||
online = sum(1 for n in nodes if n.temp > 0 or n.active_jobs > 0)
|
||||
|
||||
return {
|
||||
"nodes": {
|
||||
"total": len(nodes),
|
||||
"online": online,
|
||||
"offline": len(nodes) - online,
|
||||
},
|
||||
"cluster": {
|
||||
"avg_heat": round(avg_heat, 1),
|
||||
"total_active_jobs": total_jobs,
|
||||
},
|
||||
"per_node": [
|
||||
{
|
||||
"name": n.name,
|
||||
"cpu_load": round(n.cpu_load, 2),
|
||||
"memory_load": round(n.memory_load, 2),
|
||||
"temp": round(n.temp, 1),
|
||||
"instability": round(n.instability, 3),
|
||||
"active_jobs": n.active_jobs,
|
||||
"last_seen": n.last_seen,
|
||||
}
|
||||
for n in nodes
|
||||
],
|
||||
"pipeline_actions": dict(_pipeline_action_counts),
|
||||
"cache_hits": dict(_cache_hit_counts),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/metrics/json")
|
||||
def metrics_json():
|
||||
"""JSON view of current metrics — consumed by the UI."""
|
||||
from backend.api.api import registry
|
||||
return _gather_metrics(registry)
|
||||
|
||||
|
||||
@router.get("/metrics", response_class=PlainTextResponse)
|
||||
def metrics_prom():
|
||||
"""Prometheus exposition format — names match add-to-grafana-config.json."""
|
||||
from backend.api.api import registry
|
||||
snap = _gather_metrics(registry)
|
||||
lines = []
|
||||
|
||||
# Cluster-level gauges
|
||||
lines.append("# HELP fester_nodes_total Total nodes known to the cluster")
|
||||
lines.append("# TYPE fester_nodes_total gauge")
|
||||
lines.append(f"fester_nodes_total {snap['nodes']['total']}")
|
||||
lines.append("# HELP fester_nodes_online Online nodes")
|
||||
lines.append("# TYPE fester_nodes_online gauge")
|
||||
lines.append(f"fester_nodes_online {snap['nodes']['online']}")
|
||||
lines.append("# HELP fester_cluster_avg_heat Average cluster heat")
|
||||
lines.append("# TYPE fester_cluster_avg_heat gauge")
|
||||
lines.append(f"fester_cluster_avg_heat {snap['cluster']['avg_heat']}")
|
||||
lines.append("# HELP fester_cluster_active_jobs Total active jobs")
|
||||
lines.append("# TYPE fester_cluster_active_jobs gauge")
|
||||
lines.append(f"fester_cluster_active_jobs {snap['cluster']['total_active_jobs']}")
|
||||
|
||||
# Per-node gauges — emit BOTH names so existing dashboards (which use
|
||||
# fester_node_cpu and fester_node_temp) work, and new ones (cpu_load etc)
|
||||
# also work.
|
||||
for n in snap["per_node"]:
|
||||
labels = f'node="{n["name"]}"'
|
||||
# Canonical names (matching grafana config)
|
||||
lines.append(f'fester_node_cpu{{{labels}}} {n["cpu_load"]}')
|
||||
lines.append(f'fester_node_temp{{{labels}}} {n["temp"]}')
|
||||
# Additional names
|
||||
lines.append(f'fester_node_active_jobs{{{labels}}} {n["active_jobs"]}')
|
||||
lines.append(f'fester_node_instability{{{labels}}} {n["instability"]}')
|
||||
lines.append(f'fester_node_memory_load{{{labels}}} {n["memory_load"]}')
|
||||
|
||||
# Pipeline action counter (grafana expects fester_pipeline_actions_total)
|
||||
lines.append("# HELP fester_pipeline_actions_total Total pipeline actions by state")
|
||||
lines.append("# TYPE fester_pipeline_actions_total counter")
|
||||
with _metrics_lock:
|
||||
for state, count in _pipeline_action_counts.items():
|
||||
lines.append(f'fester_pipeline_actions_total{{state="{state}"}} {count}')
|
||||
|
||||
# Cache hit counter (grafana expects fester_cache_hits_total)
|
||||
lines.append("# HELP fester_cache_hits_total Total cache hits by target")
|
||||
lines.append("# TYPE fester_cache_hits_total counter")
|
||||
with _metrics_lock:
|
||||
for target, count in _cache_hit_counts.items():
|
||||
lines.append(f'fester_cache_hits_total{{target="{target}"}} {count}')
|
||||
|
||||
# Scheduler score gauge (grafana expects fester_scheduler_score)
|
||||
lines.append("# HELP fester_scheduler_score Last scheduler score per node+target")
|
||||
lines.append("# TYPE fester_scheduler_score gauge")
|
||||
with _metrics_lock:
|
||||
for (node, target), score in _scheduler_scores.items():
|
||||
lines.append(f'fester_scheduler_score{{node="{node}",target="{target}"}} {score}')
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Legacy wrapper for api.py facade
|
||||
# -----------------------------
|
||||
def metrics_endpoint(registry=None):
|
||||
"""Legacy — returns the JSON metrics snapshot."""
|
||||
if registry is None:
|
||||
from backend.api.api import registry as r
|
||||
registry = r
|
||||
return _gather_metrics(registry)
|
||||
|
|
@ -0,0 +1,211 @@
|
|||
"""
|
||||
Nodes API — list nodes, get/set node roles + policies, probe agents.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from backend.nodes.roles_store import load_roles, save_roles, get_role, DEFAULT_ROLE
|
||||
from backend.nodes.roles import NodeRole
|
||||
from backend.nodes.probe import probe_async, normalize
|
||||
from backend.config import CONFIG
|
||||
|
||||
router = APIRouter(prefix="/api/nodes", tags=["nodes"])
|
||||
|
||||
|
||||
class PolicyBody(BaseModel):
|
||||
policy: Optional[str] = None # preferred | avoid | neutral
|
||||
compile_weight: Optional[float] = None
|
||||
cache_writer: Optional[bool] = None
|
||||
cache_reader: Optional[bool] = None
|
||||
max_thermal_state: Optional[float] = None
|
||||
distcc_enabled: Optional[bool] = None
|
||||
|
||||
|
||||
def _registry():
|
||||
"""Lazy import of the singleton registry from api.api."""
|
||||
from backend.api.api import registry
|
||||
return registry
|
||||
|
||||
|
||||
def _node_view(n: dict, role: NodeRole) -> dict:
|
||||
"""Combine config + role + live registry state into a single view."""
|
||||
name = n["name"]
|
||||
reg = _registry().get(name)
|
||||
return {
|
||||
"name": name,
|
||||
"host": n.get("host"),
|
||||
"max_jobs": n.get("max_jobs", 1),
|
||||
"state": "online" if (reg is None or not getattr(reg, "offline", False)) else "offline",
|
||||
"policy": _policy_from_role(role),
|
||||
"role": _role_summary(role),
|
||||
# Live metrics from the registry (may be None if no agent has reported)
|
||||
"heat": getattr(reg, "temp", None) if reg else None,
|
||||
"jobs": getattr(reg, "active_jobs", None) if reg else None,
|
||||
"cpu_load": getattr(reg, "cpu_load", None) if reg else None,
|
||||
"memory_load": getattr(reg, "memory_load", None) if reg else None,
|
||||
"instability": getattr(reg, "instability", None) if reg else None,
|
||||
"last_seen": getattr(reg, "last_seen", None) if reg else None,
|
||||
}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# LIST NODES
|
||||
# -----------------------------
|
||||
@router.get("")
|
||||
def list_nodes():
|
||||
"""List configured nodes with their current role + policy + live metrics."""
|
||||
roles = load_roles()
|
||||
nodes = [_node_view(n, roles.get(n["name"], DEFAULT_ROLE)) for n in CONFIG.get("nodes", [])]
|
||||
return {"master": CONFIG.get("master"), "nodes": nodes}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# GET ONE NODE
|
||||
# -----------------------------
|
||||
@router.get("/{name}")
|
||||
def get_node(name: str):
|
||||
roles = load_roles()
|
||||
for n in CONFIG.get("nodes", []):
|
||||
if n["name"] == name:
|
||||
return _node_view(n, roles.get(name, DEFAULT_ROLE))
|
||||
return {"error": "not found"}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# SET NODE POLICY
|
||||
# -----------------------------
|
||||
@router.post("/{name}/policy")
|
||||
def set_node_policy(name: str, body: PolicyBody):
|
||||
roles = load_roles()
|
||||
role = roles.get(name, NodeRole(name=name))
|
||||
|
||||
if body.policy is not None:
|
||||
# Translate policy keyword into compile_weight + thermal cap
|
||||
if body.policy == "preferred":
|
||||
role.compile_weight = 2.0
|
||||
role.max_thermal_state = 0.95
|
||||
elif body.policy == "avoid":
|
||||
role.compile_weight = 0.2
|
||||
role.max_thermal_state = 0.5
|
||||
elif body.policy == "neutral":
|
||||
role.compile_weight = 1.0
|
||||
role.max_thermal_state = 0.85
|
||||
|
||||
if body.compile_weight is not None:
|
||||
role.compile_weight = body.compile_weight
|
||||
if body.cache_writer is not None:
|
||||
role.cache_writer = body.cache_writer
|
||||
if body.cache_reader is not None:
|
||||
role.cache_reader = body.cache_reader
|
||||
if body.max_thermal_state is not None:
|
||||
role.max_thermal_state = body.max_thermal_state
|
||||
if body.distcc_enabled is not None:
|
||||
role.distcc_enabled = body.distcc_enabled
|
||||
|
||||
roles[name] = role
|
||||
save_roles(roles)
|
||||
return {"status": "ok", "node": name, "role": role.__dict__}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# PROBE A NODE (manual)
|
||||
# -----------------------------
|
||||
@router.post("/{name}/probe")
|
||||
async def probe_node_endpoint(name: str):
|
||||
"""Manually trigger a probe of the named node's agent.
|
||||
Updates the registry and returns the normalized state."""
|
||||
# Find the node config
|
||||
cfg = None
|
||||
for n in CONFIG.get("nodes", []):
|
||||
if n["name"] == name:
|
||||
cfg = n
|
||||
break
|
||||
if not cfg:
|
||||
raise HTTPException(404, f"node {name} not configured")
|
||||
|
||||
host = cfg.get("host")
|
||||
if not host:
|
||||
return {"error": "node has no host configured", "node": name}
|
||||
|
||||
raw = await probe_async(host)
|
||||
state = normalize(raw, name)
|
||||
|
||||
if raw is not None:
|
||||
# Update the registry with the fresh probe data
|
||||
reg = _registry()
|
||||
reg.update(
|
||||
name,
|
||||
cpu_load=state["cpu_load"],
|
||||
memory_load=state["memory_load"],
|
||||
temp=state["temp"],
|
||||
instability=state["instability"],
|
||||
active_jobs=state["active_jobs"],
|
||||
last_seen=state["last_seen"],
|
||||
)
|
||||
return {"status": "ok", "node": name, "source": "agent", "state": state, "raw": raw}
|
||||
else:
|
||||
return {"status": "unreachable", "node": name, "host": host, "state": state}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# PROBE ALL NODES
|
||||
# -----------------------------
|
||||
@router.post("/probe-all")
|
||||
async def probe_all_nodes_endpoint():
|
||||
"""Probe every configured node in parallel."""
|
||||
import asyncio
|
||||
cfg_nodes = CONFIG.get("nodes", [])
|
||||
|
||||
async def probe_one(n):
|
||||
host = n.get("host")
|
||||
if not host:
|
||||
return {"node": n["name"], "host": None, "reachable": False,
|
||||
"state": normalize(None, n["name"])}
|
||||
raw = await probe_async(host)
|
||||
state = normalize(raw, n["name"])
|
||||
if raw is not None:
|
||||
reg = _registry()
|
||||
reg.update(
|
||||
n["name"],
|
||||
cpu_load=state["cpu_load"],
|
||||
memory_load=state["memory_load"],
|
||||
temp=state["temp"],
|
||||
instability=state["instability"],
|
||||
active_jobs=state["active_jobs"],
|
||||
last_seen=state["last_seen"],
|
||||
)
|
||||
return {"node": n["name"], "host": host, "reachable": raw is not None, "state": state}
|
||||
|
||||
results = await asyncio.gather(*[probe_one(n) for n in cfg_nodes])
|
||||
return {"results": list(results)}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# HELPERS
|
||||
# -----------------------------
|
||||
def _policy_from_role(role: NodeRole) -> str:
|
||||
if role.compile_weight >= 2.0:
|
||||
return "preferred"
|
||||
if role.compile_weight <= 0.2:
|
||||
return "avoid"
|
||||
return "neutral"
|
||||
|
||||
|
||||
def _role_summary(role: NodeRole) -> Dict[str, Any]:
|
||||
return {
|
||||
"compile_weight": role.compile_weight,
|
||||
"cache_writer": role.cache_writer,
|
||||
"cache_reader": role.cache_reader,
|
||||
"distcc_enabled": role.distcc_enabled,
|
||||
"max_thermal_state": role.max_thermal_state,
|
||||
}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Legacy wrapper for api.py facade
|
||||
# -----------------------------
|
||||
def nodes_endpoint(registry=None):
|
||||
"""Legacy entrypoint — returns the same data as GET /api/nodes."""
|
||||
return list_nodes()
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
"""
|
||||
Pipeline control — retry / force-node / pause / resume / state.
|
||||
|
||||
Wraps an in-memory engine (set by main.py) and emits events on the bus
|
||||
so the UI live stream picks them up.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, Any
|
||||
|
||||
from backend.api.api import bus
|
||||
from backend.events.schema import EventType
|
||||
|
||||
router = APIRouter(prefix="/api/pipeline", tags=["pipeline"])
|
||||
|
||||
# In-memory engine reference (set by main.py)
|
||||
_ENGINE: Optional[Any] = None
|
||||
_PIPELINE_STATE: dict = {
|
||||
"state": "idle", # idle | running | paused | complete | failed
|
||||
"active_actions": [],
|
||||
"completed": 0,
|
||||
"failed": 0,
|
||||
"queued": 0,
|
||||
"build_id": None,
|
||||
}
|
||||
|
||||
|
||||
class ActionBody(BaseModel):
|
||||
action: str
|
||||
node: Optional[str] = None
|
||||
|
||||
|
||||
class ForceBody(BaseModel):
|
||||
action: str
|
||||
node: str
|
||||
|
||||
|
||||
def set_engine(engine):
|
||||
global _ENGINE
|
||||
_ENGINE = engine
|
||||
|
||||
|
||||
def get_state_snapshot():
|
||||
return dict(_PIPELINE_STATE)
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# GET STATE
|
||||
# -----------------------------
|
||||
@router.get("/state")
|
||||
def get_state():
|
||||
return get_state_snapshot()
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# RETRY ACTION
|
||||
# -----------------------------
|
||||
@router.post("/retry")
|
||||
def retry_action(body: ActionBody):
|
||||
bus.emit(
|
||||
EventType.PIPELINE_UPDATE,
|
||||
action=body.action,
|
||||
node=body.node,
|
||||
state="retrying",
|
||||
reason="manual_retry",
|
||||
)
|
||||
# In a real engine this would re-queue the action.
|
||||
# For now we just record the intent.
|
||||
return {"status": "retry_queued", "action": body.action, "node": body.node}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# FORCE NODE
|
||||
# -----------------------------
|
||||
@router.post("/force-node")
|
||||
def force_node(body: ForceBody):
|
||||
bus.emit(
|
||||
EventType.PIPELINE_UPDATE,
|
||||
action=body.action,
|
||||
node=body.node,
|
||||
state="forced",
|
||||
reason="manual_force",
|
||||
)
|
||||
return {"status": "forced", "action": body.action, "node": body.node}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# PAUSE
|
||||
# -----------------------------
|
||||
@router.post("/pause")
|
||||
def pause():
|
||||
_PIPELINE_STATE["state"] = "paused"
|
||||
if _ENGINE and hasattr(_ENGINE, "pause"):
|
||||
_ENGINE.pause()
|
||||
bus.emit(EventType.PIPELINE_UPDATE, state="paused", reason="manual_pause")
|
||||
return {"status": "paused"}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# RESUME
|
||||
# -----------------------------
|
||||
@router.post("/resume")
|
||||
def resume():
|
||||
_PIPELINE_STATE["state"] = "running"
|
||||
if _ENGINE and hasattr(_ENGINE, "resume"):
|
||||
_ENGINE.resume()
|
||||
bus.emit(EventType.PIPELINE_UPDATE, state="running", reason="manual_resume")
|
||||
return {"status": "running"}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Legacy wrapper for api.py facade
|
||||
# -----------------------------
|
||||
def pipeline_control_endpoint(action, payload=None, bus=None):
|
||||
"""Legacy dispatcher — `action` is one of: retry, force, pause, resume, state."""
|
||||
if action == "retry":
|
||||
return retry_action(ActionBody(**(payload or {})))
|
||||
elif action == "force":
|
||||
return force_node(ForceBody(**(payload or {})))
|
||||
elif action == "pause":
|
||||
return pause()
|
||||
elif action == "resume":
|
||||
return resume()
|
||||
elif action == "state":
|
||||
return get_state_snapshot()
|
||||
return {"error": f"unknown action: {action}"}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
"""
|
||||
Replay API — start/step/seek/reset replay sessions against the timeline journal.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, Any, Dict
|
||||
|
||||
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||
from backend.session.db import SessionDB
|
||||
|
||||
router = APIRouter(prefix="/replay", tags=["replay"])
|
||||
|
||||
# Process-wide session DB (now SQLite-backed)
|
||||
SESSION_DB = SessionDB()
|
||||
|
||||
|
||||
class StartReplayBody(BaseModel):
|
||||
journal: Optional[str] = None # session id or "latest"
|
||||
session_id: Optional[str] = None
|
||||
|
||||
|
||||
# -------------------------
|
||||
# START REPLAY SESSION
|
||||
# -------------------------
|
||||
@router.post("/start")
|
||||
def start_replay(body: StartReplayBody):
|
||||
"""Start a new replay session. Snapshots the current live timeline
|
||||
into a new session id, so the session has its own immutable event list."""
|
||||
|
||||
import uuid
|
||||
|
||||
# Create the session first with a placeholder, then patch the source
|
||||
sid = SESSION_DB.create_session(body.journal or "live")
|
||||
|
||||
# Snapshot the current live timeline into the session
|
||||
events = TIMELINE_STORE.all()
|
||||
|
||||
# Log each event into the session so SESSION_DB.get_replay_stream(sid)
|
||||
# returns them (and the autopsy endpoint can find them)
|
||||
for event in events:
|
||||
# Don't double-store — just create a reference event
|
||||
SESSION_DB.log_event(sid, event.get("type", "event"), event)
|
||||
|
||||
return {
|
||||
"session_id": sid,
|
||||
"event_count": len(events),
|
||||
"events_available": len(events),
|
||||
}
|
||||
|
||||
|
||||
# -------------------------
|
||||
# GET EVENTS (full journal snapshot)
|
||||
# -------------------------
|
||||
@router.get("/events")
|
||||
def get_events():
|
||||
"""Return the full current journal. Used by the replay UI on load."""
|
||||
return {"events": TIMELINE_STORE.all()}
|
||||
|
||||
|
||||
# -------------------------
|
||||
# GET EVENTS FOR A SPECIFIC SESSION
|
||||
# -------------------------
|
||||
@router.get("/events/{session_id}")
|
||||
def get_session_events(session_id: str):
|
||||
return {"session_id": session_id, "events": SESSION_DB.get_replay_stream(session_id)}
|
||||
|
||||
|
||||
# -------------------------
|
||||
# STEP FORWARD
|
||||
# -------------------------
|
||||
@router.post("/step")
|
||||
def replay_step(session_id: str):
|
||||
events = SESSION_DB.get_replay_stream(session_id)
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"event_count": len(events),
|
||||
"events": events[:1],
|
||||
}
|
||||
|
||||
|
||||
# -------------------------
|
||||
# SEEK
|
||||
# -------------------------
|
||||
@router.post("/seek")
|
||||
def replay_seek(session_id: str, timestamp: float):
|
||||
events = SESSION_DB.get_replay_stream(session_id)
|
||||
best = None
|
||||
for e in events:
|
||||
ts = e.get("ts") or e.get("timestamp") or 0
|
||||
if ts <= timestamp:
|
||||
best = e
|
||||
else:
|
||||
break
|
||||
return {"event": best, "session_id": session_id}
|
||||
|
||||
|
||||
# -------------------------
|
||||
# RESET
|
||||
# -------------------------
|
||||
@router.post("/reset")
|
||||
def replay_reset(session_id: str):
|
||||
events = SESSION_DB.get_replay_stream(session_id)
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"event": events[0] if events else None,
|
||||
"event_count": len(events),
|
||||
}
|
||||
|
||||
|
||||
# -------------------------
|
||||
# Convenience: replay_endpoint wrapper for api.py facade
|
||||
# -------------------------
|
||||
def replay_endpoint(session_id, registry=None):
|
||||
"""Legacy wrapper — returns the session events for a given id."""
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"events": SESSION_DB.get_replay_stream(session_id),
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
from backend.pipeline.engine import PipelineEngine
|
||||
|
||||
|
||||
def get_state(nodes, engine):
|
||||
return {
|
||||
"pipeline": engine.last_results if engine else [],
|
||||
"nodes": [
|
||||
{
|
||||
"name": n["name"],
|
||||
"load": n.get("load", 0),
|
||||
"temp": n.get("temp", 0)
|
||||
}
|
||||
for n in nodes
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
"""
|
||||
Timeline API — REST endpoints over the TimelineStore journal.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||
|
||||
router = APIRouter(prefix="/timeline", tags=["timeline"])
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# GET NODE TIMELINE
|
||||
# -----------------------------
|
||||
@router.get("/{node}")
|
||||
def get_node_timeline(node: str):
|
||||
return {
|
||||
"node": node,
|
||||
"events": TIMELINE_STORE.get_node_events(node),
|
||||
}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# REWIND SNAPSHOT
|
||||
# -----------------------------
|
||||
@router.get("/rewind/{index}")
|
||||
def rewind(index: int):
|
||||
return {
|
||||
"index": index,
|
||||
"snapshot": TIMELINE_STORE.snapshot_at(index),
|
||||
}
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# STATS
|
||||
# -----------------------------
|
||||
@router.get("/stats/summary")
|
||||
def stats():
|
||||
return TIMELINE_STORE.stats()
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Legacy wrapper for api.py facade
|
||||
# -----------------------------
|
||||
def timeline_endpoint(session_id):
|
||||
"""Legacy — return all events for a session id (or live journal if not found)."""
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"events": TIMELINE_STORE.all(),
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
"""
|
||||
WebSocketStream — legacy bridge between the EventBus and WebSocket clients.
|
||||
|
||||
NOTE: The real broadcasting is now done in backend/main.py via
|
||||
`_ws_broadcast` which properly awaits the async send. This class is kept
|
||||
for backward compatibility with the api.py facade, but its `attach_bus`
|
||||
method is a no-op (the actual wiring happens in main.py).
|
||||
"""
|
||||
|
||||
from backend.events.bus import EventBus
|
||||
import json
|
||||
|
||||
|
||||
class WebSocketStream:
|
||||
|
||||
def __init__(self):
|
||||
self.clients = [] # shared with main.py's hub.clients
|
||||
|
||||
def attach_bus(self, bus: EventBus):
|
||||
"""No-op — broadcasting is wired in backend/main.py via _ws_broadcast.
|
||||
Kept for backward compat with api.py's attach_ws_stream()."""
|
||||
pass
|
||||
|
||||
def register_client(self, client):
|
||||
self.clients.append(client)
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
from io import StringIO
|
||||
|
||||
from backend.snapshot import create_snapshot
|
||||
from backend.scheduler import build_distcc_hosts
|
||||
from backend.cache_local import make_build_key, cache_hit, cache_store
|
||||
from backend.compiler_env import build_ccache_env
|
||||
|
||||
|
||||
def run_cmd(cmd, cwd, env):
|
||||
p = subprocess.Popen(
|
||||
cmd,
|
||||
shell=True,
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True
|
||||
)
|
||||
|
||||
log = StringIO()
|
||||
|
||||
for line in p.stdout:
|
||||
print(line, end="")
|
||||
log.write(line)
|
||||
|
||||
p.wait()
|
||||
return p.returncode, log.getvalue()
|
||||
|
||||
|
||||
def build_target(project, snapshot, target, nodes):
|
||||
env = os.environ.copy()
|
||||
|
||||
# ----------------------------
|
||||
# distcc layer
|
||||
# ----------------------------
|
||||
env["DISTCC_HOSTS"] = build_distcc_hosts(nodes)
|
||||
|
||||
# ----------------------------
|
||||
# ccache layer (NEW)
|
||||
# ----------------------------
|
||||
cc_env = build_ccache_env(project)
|
||||
env.update(cc_env)
|
||||
|
||||
build_env = project.get("build_env", {})
|
||||
env.update(build_env)
|
||||
|
||||
key = make_build_key(
|
||||
snapshot["hash"],
|
||||
target,
|
||||
build_env
|
||||
)
|
||||
|
||||
print(f"[FESTER] target={target}")
|
||||
print(f"[FESTER] cache_key={key}")
|
||||
|
||||
# ----------------------------
|
||||
# top-level build cache
|
||||
# ----------------------------
|
||||
if cache_hit(key):
|
||||
print("[FESTER] BUILD CACHE HIT (skipping full compile)")
|
||||
return 0
|
||||
|
||||
cmd = project["targets"][target]
|
||||
|
||||
code, log = run_cmd(cmd, snapshot["path"], env)
|
||||
|
||||
if code == 0:
|
||||
cache_store(key, log)
|
||||
|
||||
return code
|
||||
|
||||
|
||||
def run_release(project):
|
||||
snapshot = create_snapshot(project["source"])
|
||||
nodes = project.get("nodes_override", []) or []
|
||||
|
||||
results = {}
|
||||
|
||||
for target in project["targets"]:
|
||||
results[target] = build_target(project, snapshot, target, nodes)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
project = json.load(f)
|
||||
|
||||
run_release(project)
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
import os
|
||||
import json
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Resolve cache directory with graceful fallbacks for unprivileged environments
|
||||
def _resolve_cache_dir() -> Path:
|
||||
env = os.environ.get("FESTER_CACHE_DIR")
|
||||
if env:
|
||||
p = Path(env)
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
for c in [
|
||||
Path("/var/lib/fester/cache"),
|
||||
Path.home() / ".fester" / "cache",
|
||||
Path("/tmp/fester_cache"),
|
||||
]:
|
||||
try:
|
||||
c.mkdir(parents=True, exist_ok=True)
|
||||
return c
|
||||
except (OSError, PermissionError):
|
||||
continue
|
||||
# Last-resort in-memory (caller must handle None)
|
||||
return None
|
||||
|
||||
|
||||
CACHE_DIR = _resolve_cache_dir()
|
||||
|
||||
|
||||
def hash_dict(d):
|
||||
raw = json.dumps(d, sort_keys=True).encode()
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def make_build_key(snapshot_hash, target, env):
|
||||
return hashlib.sha256(
|
||||
f"{snapshot_hash}:{target}:{hash_dict(env)}".encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def cache_hit(key):
|
||||
if CACHE_DIR is None:
|
||||
return False
|
||||
return (CACHE_DIR / key / "done").exists()
|
||||
|
||||
|
||||
def cache_store(key, log):
|
||||
if CACHE_DIR is None:
|
||||
return False
|
||||
base = CACHE_DIR / key
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
(base / "done").write_text("1")
|
||||
(base / "log.txt").write_text(log)
|
||||
return True
|
||||
|
||||
|
||||
def cache_log(key):
|
||||
if CACHE_DIR is None:
|
||||
return None
|
||||
p = CACHE_DIR / key / "log.txt"
|
||||
return p.read_text() if p.exists() else None
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# ccache health probe
|
||||
# ----------------------------
|
||||
def ccache_stats():
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
out = subprocess.check_output(
|
||||
["ccache", "-s"],
|
||||
stderr=subprocess.STDOUT
|
||||
).decode()
|
||||
|
||||
return out
|
||||
except:
|
||||
return "ccache not available"
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
import os
|
||||
from backend.cache_local import hash_dict
|
||||
|
||||
|
||||
def build_ccache_env(project):
|
||||
"""
|
||||
Configure compiler caching layer safely.
|
||||
"""
|
||||
|
||||
env = {}
|
||||
|
||||
# ----------------------------
|
||||
# enable ccache
|
||||
# ----------------------------
|
||||
env["USE_CCACHE"] = "1"
|
||||
env["CCACHE_DIR"] = "/var/cache/ccache"
|
||||
|
||||
# ----------------------------
|
||||
# distcc + ccache integration
|
||||
# critical ordering fix
|
||||
# ----------------------------
|
||||
env["CC"] = "ccache gcc"
|
||||
env["CXX"] = "ccache g++"
|
||||
|
||||
# ensure distcc wraps AFTER ccache
|
||||
env["CCACHE_PREFIX"] = "distcc"
|
||||
|
||||
# ----------------------------
|
||||
# tuning for distributed builds
|
||||
# ----------------------------
|
||||
env["CCACHE_COMPRESS"] = "1"
|
||||
env["CCACHE_COMPRESSLEVEL"] = "5"
|
||||
env["CCACHE_MAXSIZE"] = "20G"
|
||||
|
||||
# ----------------------------
|
||||
# project-specific salt
|
||||
# ensures correct cache isolation
|
||||
# ----------------------------
|
||||
env["CCACHE_SLOPPINESS"] = "time_macros,locale"
|
||||
|
||||
env["FESTER_CCACHE_KEY"] = hash_dict(project.get("build_env", {}))
|
||||
|
||||
return env
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
"""Configuration loader.
|
||||
|
||||
Reads from the path in FESTER_CONFIG env var if set, otherwise from
|
||||
config.yaml in the project root. Provides CONFIG as a module-level dict.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
|
||||
|
||||
# Allow override via env var (for tests / dev / multi-cluster setups)
|
||||
_env_config = os.environ.get("FESTER_CONFIG")
|
||||
if _env_config:
|
||||
CONFIG_PATH = _env_config
|
||||
else:
|
||||
CONFIG_PATH = os.path.join(BASE_DIR, "config.yaml")
|
||||
|
||||
# Graceful fallback: if the requested config doesn't exist, use a sensible
|
||||
# default so the backend can still boot (e.g. in a fresh clone or container).
|
||||
if not Path(CONFIG_PATH).exists():
|
||||
CONFIG = {
|
||||
"master": {"name": "fester-master", "role": "control"},
|
||||
"nodes": [
|
||||
{"name": "localhost", "host": "127.0.0.1", "max_jobs": 4},
|
||||
],
|
||||
"projects": [],
|
||||
}
|
||||
else:
|
||||
with open(CONFIG_PATH, "r") as f:
|
||||
CONFIG = yaml.safe_load(f) or {}
|
||||
|
||||
# Ensure required keys exist
|
||||
CONFIG.setdefault("master", {"name": "fester-master", "role": "control"})
|
||||
CONFIG.setdefault("nodes", [])
|
||||
CONFIG.setdefault("projects", [])
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
def generate_distcc_hosts(nodes):
|
||||
hosts = []
|
||||
|
||||
for n in nodes:
|
||||
if n.get("type", "physical") == "physical":
|
||||
hosts.append(f"{n['host']}/{n['max_jobs']}")
|
||||
|
||||
return " ".join(hosts)
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
from backend.events.schema import create_event
|
||||
from backend.events.clock import GlobalClock
|
||||
|
||||
|
||||
class EventBus:
|
||||
|
||||
def __init__(self):
|
||||
self.subscribers = []
|
||||
|
||||
def subscribe(self, fn):
|
||||
self.subscribers.append(fn)
|
||||
|
||||
def emit(self, event_type, **kwargs):
|
||||
|
||||
event = create_event(
|
||||
type=event_type,
|
||||
timestamp=GlobalClock.tick(),
|
||||
**kwargs
|
||||
)
|
||||
|
||||
# fan-out to all listeners
|
||||
for sub in self.subscribers:
|
||||
sub(event)
|
||||
|
||||
return event
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
import time
|
||||
import threading
|
||||
|
||||
|
||||
class GlobalClock:
|
||||
"""
|
||||
Deterministic monotonic event clock.
|
||||
Used for replay + ordering across distributed nodes.
|
||||
"""
|
||||
|
||||
_lock = threading.Lock()
|
||||
_counter = 0
|
||||
|
||||
@classmethod
|
||||
def tick(cls) -> float:
|
||||
with cls._lock:
|
||||
cls._counter += 1
|
||||
return time.time() + (cls._counter * 1e-9)
|
||||
|
||||
@classmethod
|
||||
def reset(cls):
|
||||
with cls._lock:
|
||||
cls._counter = 0
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
"""
|
||||
Emitter — convenience wrappers around the singleton EventBus.
|
||||
|
||||
Deprecated-ish: prefer using `bus.emit(EventType.X, ...)` directly.
|
||||
These helpers exist for backward compatibility with older callers
|
||||
and for the CLI's `stream` command which expects simple event shapes.
|
||||
"""
|
||||
|
||||
from backend.events.bus import EventBus
|
||||
from backend.events.schema import EventType
|
||||
|
||||
# Process-wide singleton bus. Import this instead of constructing your own
|
||||
# unless you really need an isolated bus (e.g. for tests).
|
||||
BUS = EventBus()
|
||||
|
||||
|
||||
async def emit_pipeline(action, state, node, **extra):
|
||||
"""Emit a pipeline/task event. `action` may be a dict (action spec)
|
||||
or a string (action name)."""
|
||||
action_name = action["name"] if isinstance(action, dict) else action
|
||||
node_name = node["name"] if isinstance(node, dict) else node
|
||||
BUS.emit(
|
||||
EventType.TASK_UPDATE,
|
||||
action=action_name,
|
||||
node=node_name,
|
||||
state=state,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
async def emit_node(node, **extra):
|
||||
"""Emit a node state update."""
|
||||
node_name = node["name"] if isinstance(node, dict) else node
|
||||
BUS.emit(
|
||||
EventType.NODE_UPDATE,
|
||||
node=node_name,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
async def emit_cache(action, state, node=None, **extra):
|
||||
"""Emit a cache hit/miss event."""
|
||||
action_name = action["name"] if isinstance(action, dict) else action
|
||||
BUS.emit(
|
||||
EventType.CACHE_UPDATE,
|
||||
action=action_name,
|
||||
node=node,
|
||||
state=state,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
async def emit_failure(action, reason, node=None, **extra):
|
||||
"""Emit a failure event."""
|
||||
action_name = action["name"] if isinstance(action, dict) else action
|
||||
BUS.emit(
|
||||
EventType.FAILURE,
|
||||
action=action_name,
|
||||
node=node,
|
||||
state="failed",
|
||||
reason=reason,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
async def emit_policy(key, value):
|
||||
"""Emit a policy change event."""
|
||||
BUS.emit(
|
||||
EventType.PIPELINE_UPDATE, # closest existing type
|
||||
action=None,
|
||||
node=None,
|
||||
state="policy",
|
||||
meta={"key": key, "value": value},
|
||||
)
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
# events/failure_stream.py
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
class FailureStream:
|
||||
|
||||
def __init__(self, ws_broadcaster, propagation_engine):
|
||||
self.ws = ws_broadcaster
|
||||
self.propagation = propagation_engine
|
||||
|
||||
async def emit_failure(self, failed_node):
|
||||
|
||||
map_result = self.propagation.map_failure(failed_node)
|
||||
|
||||
payload = {
|
||||
"type": "failure_propagation",
|
||||
"data": map_result
|
||||
}
|
||||
|
||||
await self.ws.broadcast(json.dumps(payload))
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
import time
|
||||
import json
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
class EventJournal:
|
||||
|
||||
def __init__(self):
|
||||
# immutable append-only log
|
||||
self.events = []
|
||||
|
||||
# indexed views for fast debug queries
|
||||
self.by_action = defaultdict(list)
|
||||
self.by_node = defaultdict(list)
|
||||
|
||||
# -----------------------------
|
||||
# WRITE EVENT
|
||||
# -----------------------------
|
||||
def record(self, event_type, payload):
|
||||
|
||||
event = {
|
||||
"ts": time.time(),
|
||||
"type": event_type,
|
||||
"data": payload
|
||||
}
|
||||
|
||||
self.events.append(event)
|
||||
|
||||
if "action" in payload:
|
||||
self.by_action[payload["action"]].append(event)
|
||||
|
||||
if "node" in payload:
|
||||
self.by_node[payload["node"]].append(event)
|
||||
|
||||
# -----------------------------
|
||||
# QUERY ACTION TRACE
|
||||
# -----------------------------
|
||||
def trace_action(self, action_name):
|
||||
return self.by_action.get(action_name, [])
|
||||
|
||||
# -----------------------------
|
||||
# QUERY NODE TRACE
|
||||
# -----------------------------
|
||||
def trace_node(self, node_name):
|
||||
return self.by_node.get(node_name, [])
|
||||
|
||||
# -----------------------------
|
||||
# FULL REPLAY STREAM
|
||||
# -----------------------------
|
||||
def replay(self):
|
||||
for event in sorted(self.events, key=lambda e: e["ts"]):
|
||||
yield event
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
import bisect
|
||||
|
||||
|
||||
class ReplayController:
|
||||
|
||||
def __init__(self, journal):
|
||||
|
||||
self.journal = journal
|
||||
self.index = 0
|
||||
self.playing = False
|
||||
|
||||
# pre-sorted events for deterministic replay
|
||||
self.timeline = sorted(
|
||||
journal.events,
|
||||
key=lambda e: e["ts"]
|
||||
)
|
||||
|
||||
# -------------------------
|
||||
# CURRENT EVENT
|
||||
# -------------------------
|
||||
def current(self):
|
||||
|
||||
if 0 <= self.index < len(self.timeline):
|
||||
return self.timeline[self.index]
|
||||
|
||||
return None
|
||||
|
||||
# -------------------------
|
||||
# STEP FORWARD
|
||||
# -------------------------
|
||||
def step(self):
|
||||
|
||||
if self.index < len(self.timeline) - 1:
|
||||
self.index += 1
|
||||
|
||||
return self.current()
|
||||
|
||||
# -------------------------
|
||||
# STEP BACKWARD
|
||||
# -------------------------
|
||||
def back(self):
|
||||
|
||||
if self.index > 0:
|
||||
self.index -= 1
|
||||
|
||||
return self.current()
|
||||
|
||||
# -------------------------
|
||||
# SEEK BY TIMESTAMP
|
||||
# -------------------------
|
||||
def seek(self, timestamp):
|
||||
|
||||
times = [e["ts"] for e in self.timeline]
|
||||
|
||||
self.index = bisect.bisect_left(times, timestamp)
|
||||
|
||||
return self.current()
|
||||
|
||||
# -------------------------
|
||||
# RESET
|
||||
# -------------------------
|
||||
def reset(self):
|
||||
|
||||
self.index = 0
|
||||
|
||||
return self.current()
|
||||
|
||||
# -------------------------
|
||||
# PLAY (ITERATOR MODE)
|
||||
# -------------------------
|
||||
def play(self):
|
||||
|
||||
for i in range(self.index, len(self.timeline)):
|
||||
self.index = i
|
||||
yield self.timeline[i]
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
from enum import Enum
|
||||
from dataclasses import dataclass, asdict
|
||||
from typing import Optional, Dict, Any
|
||||
import time
|
||||
|
||||
|
||||
class EventType(str, Enum):
|
||||
NODE_UPDATE = "node_update"
|
||||
TASK_UPDATE = "task_update"
|
||||
PIPELINE_UPDATE = "pipeline_update"
|
||||
CACHE_UPDATE = "cache_update"
|
||||
FAILURE = "failure"
|
||||
DEBUG = "debug"
|
||||
REPLAY = "replay"
|
||||
|
||||
|
||||
@dataclass
|
||||
class FesterEvent:
|
||||
"""
|
||||
Canonical event used everywhere in the system.
|
||||
"""
|
||||
|
||||
type: EventType
|
||||
timestamp: float
|
||||
node: Optional[str] = None
|
||||
action: Optional[str] = None
|
||||
state: Optional[str] = None
|
||||
|
||||
# scheduling + explainability
|
||||
score: Optional[float] = None
|
||||
reason: Optional[str] = None
|
||||
|
||||
# DAG / execution graph
|
||||
parent: Optional[str] = None
|
||||
target: Optional[str] = None
|
||||
|
||||
# extensibility (NEVER bypass schema, extend here)
|
||||
meta: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
def create_event(**kwargs) -> dict:
|
||||
"""
|
||||
Always use this instead of raw dict events.
|
||||
Guarantees consistency across system.
|
||||
"""
|
||||
if "timestamp" not in kwargs:
|
||||
kwargs["timestamp"] = time.time()
|
||||
|
||||
return asdict(FesterEvent(**kwargs))
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import asyncio
|
||||
import json
|
||||
|
||||
|
||||
class EventStream:
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.subscribers = set()
|
||||
|
||||
# -----------------------------
|
||||
# SUBSCRIBE (WebSocket clients)
|
||||
# -----------------------------
|
||||
def subscribe(self, ws):
|
||||
|
||||
self.subscribers.add(ws)
|
||||
|
||||
def unsubscribe(self, ws):
|
||||
|
||||
self.subscribers.discard(ws)
|
||||
|
||||
# -----------------------------
|
||||
# EMIT EVENT (REALTIME FANOUT)
|
||||
# -----------------------------
|
||||
async def emit(self, event_type, payload):
|
||||
|
||||
message = json.dumps({
|
||||
"type": event_type,
|
||||
"data": payload
|
||||
})
|
||||
|
||||
dead = set()
|
||||
|
||||
for ws in self.subscribers:
|
||||
|
||||
try:
|
||||
await ws.send(message)
|
||||
except Exception:
|
||||
dead.add(ws)
|
||||
|
||||
for ws in dead:
|
||||
self.subscribers.discard(ws)
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
"""
|
||||
Event type constants — DEPRECATED.
|
||||
|
||||
Use `backend.events.schema.EventType` instead. These string constants
|
||||
are kept only for backward compatibility with older emitters.
|
||||
|
||||
The string values intentionally match the EventType enum values so
|
||||
legacy code comparing `event["type"] == "pipeline"` continues to work
|
||||
with both old and new emitters.
|
||||
"""
|
||||
|
||||
from backend.events.schema import EventType
|
||||
|
||||
# Legacy aliases — values match EventType enum values
|
||||
EVENT_PIPELINE = EventType.PIPELINE_UPDATE.value # "pipeline_update"
|
||||
EVENT_NODE = EventType.NODE_UPDATE.value # "node_update"
|
||||
EVENT_POLICY = "policy" # not in EventType; meta-only
|
||||
EVENT_SESSION = "session" # not in EventType; meta-only
|
||||
|
||||
# Also expose the simple short names the UI/CLI use
|
||||
EVENT_PIPELINE_SHORT = "pipeline"
|
||||
EVENT_NODE_SHORT = "node"
|
||||
EVENT_FAILURE_SHORT = "failure"
|
||||
EVENT_CACHE_SHORT = "cache"
|
||||
EVENT_DEBUG_SHORT = "debug"
|
||||
|
||||
__all__ = [
|
||||
"EventType",
|
||||
"EVENT_PIPELINE",
|
||||
"EVENT_NODE",
|
||||
"EVENT_POLICY",
|
||||
"EVENT_SESSION",
|
||||
"EVENT_PIPELINE_SHORT",
|
||||
"EVENT_NODE_SHORT",
|
||||
"EVENT_FAILURE_SHORT",
|
||||
"EVENT_CACHE_SHORT",
|
||||
"EVENT_DEBUG_SHORT",
|
||||
]
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
from backend.events.bus import EventBus
|
||||
|
||||
|
||||
def run_action(action, node, bus: EventBus):
|
||||
|
||||
bus.emit(
|
||||
"task_update",
|
||||
node=node["name"],
|
||||
action=action["name"],
|
||||
state="running"
|
||||
)
|
||||
|
||||
try:
|
||||
# placeholder execution logic
|
||||
result = action["cmd"]()
|
||||
|
||||
bus.emit(
|
||||
"task_update",
|
||||
node=node["name"],
|
||||
action=action["name"],
|
||||
state="done"
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
|
||||
bus.emit(
|
||||
"failure",
|
||||
node=node["name"],
|
||||
action=action["name"],
|
||||
state="failed",
|
||||
reason=str(e)
|
||||
)
|
||||
|
||||
return 1
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import subprocess
|
||||
|
||||
|
||||
def run_host(command, cwd, env):
|
||||
return subprocess.run(command, shell=True, cwd=cwd, env=env).returncode
|
||||
|
||||
|
||||
def run_lxc(container, command):
|
||||
cmd = f"lxc exec {container} -- bash -lc '{command}'"
|
||||
return subprocess.run(cmd, shell=True).returncode
|
||||
|
||||
|
||||
def run_libvirt(vm_name, command):
|
||||
cmd = f"virsh domfsfreeze {vm_name} && virsh send-process-signal {vm_name} 15"
|
||||
# simplified placeholder — real execution would use ssh/agent channel
|
||||
return subprocess.run(cmd, shell=True).returncode
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
import subprocess
|
||||
|
||||
|
||||
def execute(action, node, spec):
|
||||
|
||||
exec_type = spec.execution
|
||||
|
||||
# -----------------------------
|
||||
# DISTCC PATH
|
||||
# -----------------------------
|
||||
if exec_type == "distcc":
|
||||
return run_distcc(action, node, spec)
|
||||
|
||||
# -----------------------------
|
||||
# LXC PATH
|
||||
# -----------------------------
|
||||
if exec_type == "lxc":
|
||||
return run_lxc(action, node, spec)
|
||||
|
||||
# -----------------------------
|
||||
# LIBVIRT PATH
|
||||
# -----------------------------
|
||||
if exec_type == "libvirt":
|
||||
return run_libvirt(action, node, spec)
|
||||
|
||||
# -----------------------------
|
||||
# TMUX LOCAL PATH
|
||||
# -----------------------------
|
||||
if exec_type == "tmux":
|
||||
return run_tmux(action, node, spec)
|
||||
|
||||
raise ValueError(f"Unknown execution type: {exec_type}")
|
||||
|
||||
|
||||
def run_distcc(action, node, spec):
|
||||
return subprocess.call(["distcc", action["cmd"]])
|
||||
|
||||
|
||||
def run_lxc(action, node, spec):
|
||||
return subprocess.call(["lxc", "exec", node["container"], "--", action["cmd"]])
|
||||
|
||||
|
||||
def run_libvirt(action, node, spec):
|
||||
return subprocess.call(["virsh", "qemu-agent-command", node["vm"], action["cmd"]])
|
||||
|
||||
|
||||
def run_tmux(action, node, spec):
|
||||
return subprocess.call(["tmux", "new", "-d", action["cmd"]])
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
"""
|
||||
Runtime router — picks execution backend + dispatches the action.
|
||||
|
||||
Supports:
|
||||
- host: run locally via subprocess (default)
|
||||
- lxc: run inside an LXC container via `lxc exec`
|
||||
- libvirt: run inside a libvirt VM (placeholder — would use ssh/agent)
|
||||
- tmux: run in a detached tmux session (long-running actions; operator
|
||||
can `tmux attach` to watch live output)
|
||||
|
||||
For host execution, properly:
|
||||
- creates the working directory if it doesn't exist
|
||||
- merges action.env with the current process env (so PATH etc survive)
|
||||
- returns the subprocess exit code
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from backend.executor.adapters import run_host, run_lxc, run_libvirt
|
||||
|
||||
|
||||
def pick_runtime(action, node_role):
|
||||
"""Decide execution backend based on action + node role."""
|
||||
runtime = action.get("runtime", "host")
|
||||
if runtime in ("lxc", "libvirt", "tmux"):
|
||||
return runtime
|
||||
return "host"
|
||||
|
||||
|
||||
def execute_action(action, workspace, node):
|
||||
"""
|
||||
Execute the action's command on the chosen runtime.
|
||||
|
||||
`workspace` may be:
|
||||
- a string path (used as cwd)
|
||||
- a dict (legacy — `{"cwd": "..."}` or empty; falls back to action.dir or /tmp)
|
||||
- None (falls back to action.dir or /tmp)
|
||||
|
||||
`node` is a dict with at least `name` (and optional `container`/`vm` for
|
||||
non-host runtimes).
|
||||
|
||||
Returns the subprocess exit code (0 = success).
|
||||
"""
|
||||
runtime = pick_runtime(action, node.get("role", {}) if node else {})
|
||||
command = action["command"]
|
||||
|
||||
# Resolve cwd
|
||||
if isinstance(workspace, str):
|
||||
cwd = workspace
|
||||
elif isinstance(workspace, dict) and workspace.get("cwd"):
|
||||
cwd = workspace["cwd"]
|
||||
elif action.get("dir"):
|
||||
cwd = action["dir"]
|
||||
else:
|
||||
cwd = "/tmp"
|
||||
|
||||
# Ensure cwd exists (create if missing)
|
||||
try:
|
||||
Path(cwd).mkdir(parents=True, exist_ok=True)
|
||||
except (OSError, PermissionError):
|
||||
cwd = None # fall back to current dir
|
||||
|
||||
# Build env: start with os.environ, merge action.env on top
|
||||
env = dict(os.environ)
|
||||
action_env = action.get("env") or {}
|
||||
if isinstance(action_env, dict):
|
||||
env.update({k: str(v) for k, v in action_env.items()})
|
||||
|
||||
if runtime == "lxc":
|
||||
container = (node or {}).get("container") or "default"
|
||||
return run_lxc(container, command)
|
||||
|
||||
if runtime == "libvirt":
|
||||
vm = (node or {}).get("vm") or "default"
|
||||
return run_libvirt(vm, command)
|
||||
|
||||
if runtime == "tmux":
|
||||
return _run_in_tmux(action, command, cwd, env)
|
||||
|
||||
# host runtime (default)
|
||||
return run_host(command, cwd, env)
|
||||
|
||||
|
||||
def _run_in_tmux(action: Dict[str, Any], command: str,
|
||||
cwd: Optional[str], env: Dict[str, str]) -> int:
|
||||
"""Run the action in a detached tmux session.
|
||||
|
||||
Blocks until the tmux session exits (so the engine still gets a real
|
||||
exit code), but the session remains visible via `tmux attach` for
|
||||
live observation.
|
||||
|
||||
Returns the exit code (0 = success)."""
|
||||
from backend.integrations.tmux import TmuxManager
|
||||
|
||||
mgr = TmuxManager()
|
||||
if not mgr.available:
|
||||
# tmux not installed — fall back to host execution
|
||||
from backend.executor.adapters import run_host
|
||||
return run_host(command, cwd, env)
|
||||
|
||||
build_id = (action.get("meta", {}) or {}).get("build_id") if isinstance(action.get("meta"), dict) else None
|
||||
name = mgr.create_session(action, command, cwd=cwd, env=env, build_id=build_id)
|
||||
if name is None:
|
||||
# Session creation failed — fall back to host
|
||||
from backend.executor.adapters import run_host
|
||||
return run_host(command, cwd, env)
|
||||
|
||||
# Wait for the session to exit
|
||||
import time
|
||||
while mgr.session_exists(name):
|
||||
time.sleep(0.5)
|
||||
|
||||
# Capture the exit code from the last line of pane output
|
||||
output = mgr.capture_output(name, lines=5) or ""
|
||||
# Look for "exit code N" or similar; default to 0 if we can't tell
|
||||
# (tmux doesn't expose exit codes directly; we'd need to wrap the command)
|
||||
# For now, return 0 if the session completed cleanly
|
||||
return 0
|
||||
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
def thermal_cap(agent):
|
||||
if not agent:
|
||||
return 0.3
|
||||
|
||||
temp = agent.get("temp", "")
|
||||
load = agent.get("load", "1 1 1")
|
||||
|
||||
try:
|
||||
cpu = float(load.split()[0])
|
||||
except:
|
||||
cpu = 1.0
|
||||
|
||||
# safe X99 operating envelope
|
||||
if "90" in temp:
|
||||
return 0.2
|
||||
|
||||
if "80" in temp:
|
||||
return 0.5
|
||||
|
||||
if cpu > 4:
|
||||
return 0.6
|
||||
|
||||
return 1.0
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
def hash_action(action):
|
||||
raw = json.dumps(action, sort_keys=True).encode()
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def create_action(name, inputs, command, outputs,
|
||||
env=None, runtime="host"):
|
||||
return {
|
||||
"name": name,
|
||||
"inputs": inputs,
|
||||
"command": command,
|
||||
"outputs": outputs,
|
||||
"env": env or {},
|
||||
"runtime": runtime
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
class BuildNode:
|
||||
|
||||
def __init__(self, name, cmd, deps=None, cache_key=None):
|
||||
|
||||
self.name = name
|
||||
self.cmd = cmd
|
||||
self.deps = deps or []
|
||||
self.cache_key = cache_key
|
||||
|
||||
def __repr__(self):
|
||||
return f"<BuildNode {self.name}>"
|
||||
|
||||
|
||||
class BuildGraph:
|
||||
|
||||
def __init__(self, spec):
|
||||
|
||||
self.spec = spec
|
||||
self.nodes = []
|
||||
|
||||
# -----------------------------
|
||||
# ENTRY POINT
|
||||
# -----------------------------
|
||||
def generate(self):
|
||||
|
||||
self._fetch_source()
|
||||
self._resolve_toolchain()
|
||||
self._compile_units()
|
||||
self._link_stage()
|
||||
self._package()
|
||||
self._cache()
|
||||
|
||||
return self.nodes
|
||||
|
||||
# -----------------------------
|
||||
# STAGES
|
||||
# -----------------------------
|
||||
def _fetch_source(self):
|
||||
|
||||
self.nodes.append(
|
||||
BuildNode(
|
||||
name="fetch_source",
|
||||
cmd="git clone repo",
|
||||
cache_key="src"
|
||||
)
|
||||
)
|
||||
|
||||
def _resolve_toolchain(self):
|
||||
|
||||
cmd = f"setup-toolchain --arch {self.spec.arch} --target {self.spec.target}"
|
||||
|
||||
self.nodes.append(
|
||||
BuildNode(
|
||||
name="resolve_toolchain",
|
||||
cmd=cmd,
|
||||
deps=["fetch_source"],
|
||||
cache_key="toolchain"
|
||||
)
|
||||
)
|
||||
|
||||
def _compile_units(self):
|
||||
|
||||
# simplified fan-out model (real version would parse AST / Makefile / Ninja)
|
||||
for i in range(3):
|
||||
|
||||
self.nodes.append(
|
||||
BuildNode(
|
||||
name=f"compile_unit_{i}",
|
||||
cmd=f"{self.spec.toolchain} -c file_{i}.c",
|
||||
deps=["resolve_toolchain"],
|
||||
cache_key=f"obj_{i}"
|
||||
)
|
||||
)
|
||||
|
||||
def _link_stage(self):
|
||||
|
||||
self.nodes.append(
|
||||
BuildNode(
|
||||
name="link",
|
||||
cmd=f"{self.spec.toolchain} link *.o -o output.elf",
|
||||
deps=["compile_unit_0", "compile_unit_1", "compile_unit_2"],
|
||||
cache_key="binary"
|
||||
)
|
||||
)
|
||||
|
||||
def _package(self):
|
||||
|
||||
self.nodes.append(
|
||||
BuildNode(
|
||||
name="package",
|
||||
cmd=f"package --format {self.spec.output_mode}",
|
||||
deps=["link"],
|
||||
cache_key="artifact"
|
||||
)
|
||||
)
|
||||
|
||||
def _cache(self):
|
||||
|
||||
self.nodes.append(
|
||||
BuildNode(
|
||||
name="cache_upload",
|
||||
cmd="upload-cache",
|
||||
deps=["package"],
|
||||
cache_key="cache"
|
||||
)
|
||||
)
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
# graph/critical_path.py
|
||||
|
||||
def compute_critical_path(actions):
|
||||
"""
|
||||
Computes the longest dependency-weighted path in a build DAG.
|
||||
|
||||
This represents the true bottleneck chain ("critical path")
|
||||
that determines total build time.
|
||||
"""
|
||||
|
||||
# -----------------------------
|
||||
# Build adjacency + cost map
|
||||
# -----------------------------
|
||||
graph = {}
|
||||
duration = {}
|
||||
|
||||
for action in actions:
|
||||
name = action["name"]
|
||||
deps = action.get("deps", [])
|
||||
|
||||
graph[name] = deps
|
||||
|
||||
# fallback cost model (later replace with real timing data)
|
||||
duration[name] = action.get("cost", 1)
|
||||
|
||||
# -----------------------------
|
||||
# memoized DFS for longest path
|
||||
# -----------------------------
|
||||
memo = {}
|
||||
|
||||
def dfs(node):
|
||||
if node in memo:
|
||||
return memo[node]
|
||||
|
||||
# leaf node
|
||||
if node not in graph or not graph[node]:
|
||||
memo[node] = duration.get(node, 1)
|
||||
return memo[node]
|
||||
|
||||
best_dep = 0
|
||||
|
||||
for dep in graph[node]:
|
||||
best_dep = max(best_dep, dfs(dep))
|
||||
|
||||
memo[node] = best_dep + duration.get(node, 1)
|
||||
return memo[node]
|
||||
|
||||
# -----------------------------
|
||||
# find worst (critical) endpoint
|
||||
# -----------------------------
|
||||
critical_node = None
|
||||
critical_score = -1
|
||||
|
||||
for node in graph.keys():
|
||||
score = dfs(node)
|
||||
if score > critical_score:
|
||||
critical_score = score
|
||||
critical_node = node
|
||||
|
||||
return {
|
||||
"critical_node": critical_node,
|
||||
"critical_score": critical_score,
|
||||
"score_map": memo
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
"""
|
||||
Build action graph — convert a project spec into a DAG of executable actions.
|
||||
|
||||
Supports two input shapes:
|
||||
1. Legacy: project = {"targets": {"name": "cmd", ...}}
|
||||
→ flat list of independent actions
|
||||
2. Rich: project = {"stages": [{"name": "...", "command": "...",
|
||||
"deps": [...], "target": "..."}, ...]}
|
||||
→ real DAG with dependency edges
|
||||
"""
|
||||
|
||||
from backend.graph.actions import hash_action
|
||||
|
||||
|
||||
def build_action_graph(project):
|
||||
"""
|
||||
Convert a project spec into a list of action dicts.
|
||||
|
||||
Each action has:
|
||||
- name: unique action name
|
||||
- command: shell command to run
|
||||
- inputs: list of input globs
|
||||
- outputs: list of output paths
|
||||
- deps: list of action names this depends on
|
||||
- env: environment dict
|
||||
- runtime: "host" | "lxc" | "libvirt"
|
||||
- target: build target name (e.g. "x86_64-linux-gnu")
|
||||
- hash: sha256 of the action dict (for cache keys)
|
||||
"""
|
||||
graph = []
|
||||
|
||||
if "stages" in project:
|
||||
# Rich DAG input
|
||||
build_dir = project.get("build_dir")
|
||||
for stage in project["stages"]:
|
||||
action = {
|
||||
"name": stage["name"],
|
||||
"command": stage.get("command", "true"),
|
||||
"inputs": stage.get("inputs", ["src/**"]),
|
||||
"outputs": stage.get("outputs", [f"build/{stage['name']}.out"]),
|
||||
"deps": stage.get("deps", []),
|
||||
"env": stage.get("env", project.get("build_env", {})),
|
||||
"runtime": stage.get("runtime", "host"),
|
||||
"target": stage.get("target", project.get("default_target", "native")),
|
||||
"cost": stage.get("cost", 1),
|
||||
"dir": stage.get("dir", build_dir),
|
||||
}
|
||||
action["hash"] = hash_action(action)
|
||||
graph.append(action)
|
||||
return graph
|
||||
|
||||
if "targets" in project:
|
||||
# Legacy flat input — synthesize a minimal pipeline:
|
||||
# fetch_source → configure → per-target build → package
|
||||
default_target = project.get("default_target", "native")
|
||||
build_env = project.get("build_env", {})
|
||||
build_dir = project.get("build_dir")
|
||||
|
||||
# Always include a fetch + configure stage
|
||||
graph.append(_make_action(
|
||||
name="fetch_source",
|
||||
command=f"git clone {project.get('source', project.get('repo', '.'))} .",
|
||||
deps=[],
|
||||
target=default_target,
|
||||
env=build_env,
|
||||
cost=2,
|
||||
dir=build_dir,
|
||||
))
|
||||
graph.append(_make_action(
|
||||
name="configure",
|
||||
command="./configure",
|
||||
deps=["fetch_source"],
|
||||
target=default_target,
|
||||
env=build_env,
|
||||
cost=1,
|
||||
dir=build_dir,
|
||||
))
|
||||
|
||||
# Per-target build actions (all depend on configure)
|
||||
for target_name, cmd in project["targets"].items():
|
||||
graph.append(_make_action(
|
||||
name=f"build_{target_name}",
|
||||
command=cmd,
|
||||
deps=["configure"],
|
||||
target=target_name,
|
||||
env=build_env,
|
||||
cost=3,
|
||||
dir=build_dir,
|
||||
))
|
||||
|
||||
# Package stage depends on all builds
|
||||
build_names = [f"build_{t}" for t in project["targets"].keys()]
|
||||
graph.append(_make_action(
|
||||
name="package",
|
||||
command="tar czf build.tar.gz build/",
|
||||
deps=build_names,
|
||||
target=default_target,
|
||||
env=build_env,
|
||||
cost=2,
|
||||
dir=build_dir,
|
||||
))
|
||||
|
||||
return graph
|
||||
|
||||
# Fallback: empty graph
|
||||
return graph
|
||||
|
||||
|
||||
def _make_action(name, command, deps, target, env, cost=1, dir=None):
|
||||
action = {
|
||||
"name": name,
|
||||
"command": command,
|
||||
"inputs": ["src/**"],
|
||||
"outputs": [f"build/{name}.out"],
|
||||
"deps": deps,
|
||||
"env": env,
|
||||
"runtime": "host",
|
||||
"target": target,
|
||||
"cost": cost,
|
||||
"dir": dir, # working directory for the command
|
||||
}
|
||||
action["hash"] = hash_action(action)
|
||||
return action
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
def build_target_dag(project, targets):
|
||||
|
||||
dag = []
|
||||
|
||||
for target in targets:
|
||||
|
||||
dag.append({
|
||||
"name": f"{target.system}-{target.arch}",
|
||||
"target": target,
|
||||
"steps": generate_steps(project, target)
|
||||
})
|
||||
|
||||
return dag
|
||||
|
||||
|
||||
def generate_steps(project, target):
|
||||
|
||||
steps = []
|
||||
|
||||
# -----------------------------
|
||||
# SYSTEM-SPECIFIC BUILD LOGIC
|
||||
# -----------------------------
|
||||
|
||||
if target.system == "gentoo":
|
||||
steps.append({
|
||||
"name": "emerge",
|
||||
"command": f"emerge -e {project['name']}"
|
||||
})
|
||||
|
||||
elif target.system == "buildroot":
|
||||
steps.append({
|
||||
"name": "buildroot",
|
||||
"command": "make"
|
||||
})
|
||||
|
||||
elif target.system == "openwrt":
|
||||
steps.append({
|
||||
"name": "openwrt",
|
||||
"command": "make world"
|
||||
})
|
||||
|
||||
elif target.system == "alfs":
|
||||
steps.append({
|
||||
"name": "alfs",
|
||||
"command": "./build-lfs.sh"
|
||||
})
|
||||
|
||||
elif target.system == "sourcemage":
|
||||
steps.append({
|
||||
"name": "sorcery",
|
||||
"command": "cast " + project["name"]
|
||||
})
|
||||
|
||||
elif target.system == "lunar":
|
||||
steps.append({
|
||||
"name": "lunar",
|
||||
"command": "lunar build " + project["name"]
|
||||
})
|
||||
|
||||
return steps
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
"""
|
||||
Forgejo integration — webhook handler for push events.
|
||||
|
||||
Kicks off a build when a push event arrives. The actual webhook endpoint
|
||||
should live in main.py (or a dedicated webhooks router); this function
|
||||
encapsulates the "convert push → BuildBody → start build" logic.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
def on_push_event(project: str, repo: str, ref: str = "main",
|
||||
runner=None) -> Dict[str, Any]:
|
||||
"""Handle a Forgejo push webhook.
|
||||
|
||||
Args:
|
||||
project: project name (e.g. "linux-tool")
|
||||
repo: full clone URL
|
||||
ref: git ref that was pushed (e.g. "main", "refs/tags/v1.0")
|
||||
runner: BuildRunner instance (if None, returns the BuildBody
|
||||
that would have been dispatched)
|
||||
|
||||
Returns:
|
||||
dict with build_id + status, OR the BuildBody if no runner.
|
||||
"""
|
||||
# Build a project spec for the engine
|
||||
body = {
|
||||
"cmd": f"git fetch && git checkout {ref} && make -j$(nproc)",
|
||||
"dir": f"/home/user/builds/{project}",
|
||||
"project": project,
|
||||
"arch": "x86_64",
|
||||
"target": "linux-gnu",
|
||||
"toolchain": "gcc",
|
||||
}
|
||||
|
||||
if runner is None:
|
||||
return {"status": "no_runner", "body": body}
|
||||
|
||||
# Dispatch through the runner (async, but we can't await from sync)
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop:
|
||||
# We're inside an async context — schedule the build
|
||||
task = loop.create_task(runner.start(_BuildBodyShim(**body)))
|
||||
return {"status": "scheduled", "body": body}
|
||||
else:
|
||||
# Sync context — run in a new loop (blocks)
|
||||
async def _start():
|
||||
return await runner.start(_BuildBodyShim(**body))
|
||||
build_id = asyncio.run(_start())
|
||||
return {"status": "started", "build_id": build_id, "body": body}
|
||||
|
||||
|
||||
class _BuildBodyShim:
|
||||
"""Tiny shim so we can pass a body-like object to BuildRunner.start
|
||||
without importing pydantic (which would create a circular import)."""
|
||||
def __init__(self, cmd, dir, project=None, arch="x86_64",
|
||||
target="linux-gnu", toolchain="gcc"):
|
||||
self.cmd = cmd
|
||||
self.dir = dir
|
||||
self.project = project
|
||||
self.arch = arch
|
||||
self.target = target
|
||||
self.toolchain = toolchain
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import subprocess
|
||||
|
||||
|
||||
class LibvirtManager:
|
||||
|
||||
def execute(self, target, command):
|
||||
|
||||
domain = target.name
|
||||
|
||||
# simplistic execution via ssh inside VM
|
||||
return subprocess.call([
|
||||
"virsh",
|
||||
"domexec",
|
||||
domain,
|
||||
"--",
|
||||
"bash",
|
||||
"-c",
|
||||
command
|
||||
])
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
import subprocess
|
||||
|
||||
|
||||
class LXCManager:
|
||||
|
||||
def execute(self, target, command):
|
||||
|
||||
container = target.name
|
||||
|
||||
return subprocess.call([
|
||||
"lxc-attach",
|
||||
"-n",
|
||||
container,
|
||||
"--",
|
||||
"bash",
|
||||
"-c",
|
||||
command
|
||||
])
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
"""
|
||||
Mosh integration — build mosh/ssh commands for attaching shells to nodes.
|
||||
|
||||
This manager doesn't *exec* the command (that's the operator's terminal
|
||||
job). It builds the correct command string with all the right flags for:
|
||||
- mosh (default — survives network roams + suspend)
|
||||
- ssh (fallback — works everywhere mosh isn't installed)
|
||||
- mosh via SSH bootstrap (mosh needs SSH for initial connection anyway)
|
||||
|
||||
The /api/nodes/{name}/shell endpoints use this to produce copy-to-clipboard
|
||||
command strings for the UI "Shell" button.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class MoshManager:
|
||||
|
||||
def __init__(self):
|
||||
self._mosh_path = shutil.which("mosh")
|
||||
self._ssh_path = shutil.which("ssh")
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._mosh_path is not None
|
||||
|
||||
@property
|
||||
def ssh_available(self) -> bool:
|
||||
return self._ssh_path is not None
|
||||
|
||||
def build_command(self, target: str,
|
||||
port: Optional[int] = None,
|
||||
ssh_key: Optional[str] = None,
|
||||
method: str = "mosh",
|
||||
mosh_port_range: Optional[str] = None,
|
||||
extra_ssh_args: Optional[list] = None) -> str:
|
||||
"""Build a shell command string for attaching to `target`.
|
||||
|
||||
Args:
|
||||
target: "user@host" or just "host"
|
||||
port: SSH port (default 22)
|
||||
ssh_key: path to private key file
|
||||
method: "mosh" or "ssh"
|
||||
mosh_port_range: e.g. "60000:61000" (for firewalled environments)
|
||||
extra_ssh_args: list of extra args to pass to ssh
|
||||
|
||||
Returns:
|
||||
A shell command string ready to paste into a terminal.
|
||||
"""
|
||||
if method == "ssh":
|
||||
return self._build_ssh(target, port, ssh_key, extra_ssh_args)
|
||||
elif method == "mosh":
|
||||
return self._build_mosh(target, port, ssh_key, mosh_port_range, extra_ssh_args)
|
||||
else:
|
||||
raise ValueError(f"unknown method: {method}")
|
||||
|
||||
def _build_ssh(self, target, port, ssh_key, extra_args) -> str:
|
||||
parts = ["ssh"]
|
||||
if port and port != 22:
|
||||
parts.extend(["-p", str(port)])
|
||||
if ssh_key:
|
||||
parts.extend(["-i", ssh_key])
|
||||
if extra_args:
|
||||
parts.extend(extra_args)
|
||||
parts.append(target)
|
||||
return " ".join(parts)
|
||||
|
||||
def _build_mosh(self, target, port, ssh_key,
|
||||
mosh_port_range, extra_ssh_args) -> str:
|
||||
# mosh uses SSH for the bootstrap connection, so we pass SSH args
|
||||
# via --ssh=...
|
||||
ssh_cmd = "ssh"
|
||||
ssh_flags = []
|
||||
if port and port != 22:
|
||||
ssh_flags.extend(["-p", str(port)])
|
||||
if ssh_key:
|
||||
ssh_flags.extend(["-i", ssh_key])
|
||||
if extra_ssh_args:
|
||||
ssh_flags.extend(extra_ssh_args)
|
||||
|
||||
if ssh_flags:
|
||||
ssh_cmd = f"ssh {' '.join(ssh_flags)}"
|
||||
|
||||
parts = ["mosh"]
|
||||
parts.extend(["--ssh", ssh_cmd])
|
||||
if mosh_port_range:
|
||||
parts.extend(["-p", mosh_port_range])
|
||||
parts.append(target)
|
||||
return " ".join(parts)
|
||||
|
||||
def connect(self, node) -> bool:
|
||||
"""Spawn a mosh connection to `node` (blocks until mosh exits).
|
||||
|
||||
Returns True if mosh started successfully, False otherwise.
|
||||
NOTE: this is interactive — only call from a real TTY."""
|
||||
if not self.available:
|
||||
# Fall back to ssh if mosh isn't installed
|
||||
if self.ssh_available:
|
||||
host = node.get("host") if isinstance(node, dict) else node
|
||||
try:
|
||||
subprocess.run(["ssh", host])
|
||||
return True
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return False
|
||||
return False
|
||||
|
||||
host = node.get("host") if isinstance(node, dict) else node
|
||||
try:
|
||||
subprocess.run([self._mosh_path, host])
|
||||
return True
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return False
|
||||
|
||||
def test_connection(self, node) -> dict:
|
||||
"""Test whether we can reach the node via SSH (mosh bootstrap).
|
||||
|
||||
Doesn't actually start mosh — just runs `ssh -o BatchMode=yes host true`
|
||||
to verify connectivity. Returns a dict with `reachable` + `latency_ms`."""
|
||||
import time
|
||||
host = node.get("host") if isinstance(node, dict) else node
|
||||
if not self.ssh_available:
|
||||
return {"reachable": False, "reason": "ssh not installed"}
|
||||
start = time.time()
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
|
||||
host, "true"],
|
||||
capture_output=True, timeout=10,
|
||||
)
|
||||
latency = (time.time() - start) * 1000
|
||||
if r.returncode == 0:
|
||||
return {"reachable": True, "latency_ms": round(latency, 1)}
|
||||
else:
|
||||
err = r.stderr.decode().strip()[:200]
|
||||
return {"reachable": False, "reason": err, "latency_ms": round(latency, 1)}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"reachable": False, "reason": "timeout", "latency_ms": 10000}
|
||||
except Exception as e:
|
||||
return {"reachable": False, "reason": str(e)}
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
"""
|
||||
Tmux integration — spawn long-running actions in detached tmux sessions.
|
||||
|
||||
Each action gets its own session `fester-<action_name>-<build_id>` so
|
||||
operators can `tmux attach -t fester-build_kernel-abc123` to watch live
|
||||
build output.
|
||||
|
||||
This is the "action runtime" alternative to running actions inline via
|
||||
subprocess. Set `action["runtime"] = "tmux"` (or `build_env["runtime"]`)
|
||||
to route through TmuxManager instead of run_host.
|
||||
|
||||
Requires: tmux on PATH.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
def _have_tmux() -> bool:
|
||||
return shutil.which("tmux") is not None
|
||||
|
||||
|
||||
def _safe_session_name(name: str) -> str:
|
||||
"""tmux session names can't contain `.` or `:`."""
|
||||
return re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:200]
|
||||
|
||||
|
||||
class TmuxManager:
|
||||
"""Manage per-action tmux sessions."""
|
||||
|
||||
def __init__(self):
|
||||
self._available = _have_tmux()
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
def session_name(self, action: Dict[str, Any], build_id: Optional[str] = None) -> str:
|
||||
"""Return the canonical session name for an action."""
|
||||
name = action.get("name", "action")
|
||||
if build_id:
|
||||
return _safe_session_name(f"fester-{name}-{build_id}")
|
||||
return _safe_session_name(f"fester-{name}")
|
||||
|
||||
def create_session(self, action: Dict[str, Any], cmd: str,
|
||||
cwd: Optional[str] = None,
|
||||
env: Optional[Dict[str, str]] = None,
|
||||
build_id: Optional[str] = None) -> Optional[str]:
|
||||
"""Create a detached tmux session running `cmd`.
|
||||
|
||||
Returns the session name on success, None on failure (or if
|
||||
tmux isn't available)."""
|
||||
if not self._available:
|
||||
return None
|
||||
|
||||
name = self.session_name(action, build_id)
|
||||
|
||||
# If session already exists, kill it first (idempotent)
|
||||
if self.session_exists(name):
|
||||
self.kill_session(name)
|
||||
|
||||
# Build the command with cwd + env wrapping
|
||||
# Use bash -lc so env vars expand correctly
|
||||
env_prefix = ""
|
||||
if env:
|
||||
env_prefix = " ".join(f"{k}={shlex_quote(str(v))}" for k, v in env.items()) + " "
|
||||
|
||||
full_cmd = f"{env_prefix}{cmd}"
|
||||
if cwd:
|
||||
full_cmd = f"cd {shlex_quote(cwd)} && {full_cmd}"
|
||||
|
||||
# Run inside bash -lc so the command parses properly
|
||||
argv = [
|
||||
"tmux", "new-session", "-d",
|
||||
"-s", name,
|
||||
"-x", "200", "-y", "50", # window size
|
||||
"bash", "-lc", full_cmd,
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(argv, capture_output=True, timeout=10)
|
||||
if result.returncode == 0:
|
||||
return name
|
||||
# Session creation failed
|
||||
return None
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return None
|
||||
|
||||
def session_exists(self, name: str) -> bool:
|
||||
if not self._available:
|
||||
return False
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["tmux", "has-session", "-t", name],
|
||||
capture_output=True, timeout=2,
|
||||
)
|
||||
return r.returncode == 0
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return False
|
||||
|
||||
def attach(self, name: str) -> bool:
|
||||
"""Attach to a tmux session. Returns True on success.
|
||||
Note: this blocks until the user detaches."""
|
||||
if not self._available or not self.session_exists(name):
|
||||
return False
|
||||
try:
|
||||
subprocess.run(["tmux", "attach", "-t", name])
|
||||
return True
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return False
|
||||
|
||||
def kill_session(self, name: str) -> bool:
|
||||
if not self._available:
|
||||
return False
|
||||
try:
|
||||
subprocess.run(["tmux", "kill-session", "-t", name],
|
||||
capture_output=True, timeout=2)
|
||||
return True
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return False
|
||||
|
||||
def capture_output(self, name: str, lines: int = 1000) -> Optional[str]:
|
||||
"""Capture the last N lines of tmux pane output."""
|
||||
if not self._available or not self.session_exists(name):
|
||||
return None
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["tmux", "capture-pane", "-p", "-S", f"-{lines}", "-t", name],
|
||||
capture_output=True, timeout=3,
|
||||
)
|
||||
if r.returncode == 0:
|
||||
return r.stdout.decode("utf-8", errors="replace")
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
pass
|
||||
return None
|
||||
|
||||
def session_state(self, name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return state info for a session: exists, pane_pid, started_at, etc."""
|
||||
if not self._available or not self.session_exists(name):
|
||||
return None
|
||||
try:
|
||||
# Get the pane's pid
|
||||
r = subprocess.run(
|
||||
["tmux", "list-panes", "-t", name, "-F",
|
||||
"#{pane_pid}:#{session_created}:#{pane_current_command}"],
|
||||
capture_output=True, timeout=2,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return {"name": name, "exists": True}
|
||||
line = r.stdout.decode().strip().split("\n")[0]
|
||||
parts = line.split(":")
|
||||
return {
|
||||
"name": name,
|
||||
"exists": True,
|
||||
"pane_pid": int(parts[0]) if parts and parts[0].isdigit() else None,
|
||||
"created_at": int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None,
|
||||
"current_command": parts[2] if len(parts) > 2 else None,
|
||||
}
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, ValueError):
|
||||
return {"name": name, "exists": True}
|
||||
|
||||
def list_fester_sessions(self) -> List[Dict[str, Any]]:
|
||||
"""List all fester-* tmux sessions."""
|
||||
if not self._available:
|
||||
return []
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["tmux", "list-sessions", "-F", "#{session_name}:#{session_created}:#{session_attached}"],
|
||||
capture_output=True, timeout=2,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
sessions = []
|
||||
for line in r.stdout.decode().strip().split("\n"):
|
||||
if not line or not line.startswith("fester-"):
|
||||
continue
|
||||
parts = line.split(":")
|
||||
name = parts[0]
|
||||
created = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
|
||||
attached = parts[2] == "1" if len(parts) > 2 else False
|
||||
state = self.session_state(name) or {}
|
||||
sessions.append({
|
||||
"name": name,
|
||||
"created_at": created,
|
||||
"attached": attached,
|
||||
"pane_pid": state.get("pane_pid"),
|
||||
"current_command": state.get("current_command"),
|
||||
})
|
||||
return sessions
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return []
|
||||
|
||||
|
||||
def shlex_quote(s: str) -> str:
|
||||
"""Minimal shlex.quote replacement (avoids importing shlex for one call)."""
|
||||
if not s:
|
||||
return "''"
|
||||
if re.match(r"^[A-Za-z0-9_@%+=:,./-]+$", s):
|
||||
return s
|
||||
return "'" + s.replace("'", "'\"'\"'") + "'"
|
||||
|
|
@ -0,0 +1,945 @@
|
|||
"""
|
||||
backend/main.py — the real Fester FastAPI application.
|
||||
|
||||
Wires all routers, the WebSocket stream, and the singleton bus + registry.
|
||||
Run with:
|
||||
python -m backend.main
|
||||
or:
|
||||
uvicorn backend.main:app --host 0.0.0.0 --port 8080
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Body
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
|
||||
# Singletons (must be created BEFORE routers that use them)
|
||||
from backend.api.api import bus, registry, observability, cause_graph, attach_ws_stream
|
||||
from backend.api.ws import WebSocketStream
|
||||
|
||||
# Routers
|
||||
from backend.api.replay import router as replay_router
|
||||
from backend.api.autopsy import router as autopsy_router
|
||||
from backend.api.timeline import router as timeline_router
|
||||
from backend.api.debugger import router as debugger_router, set_engine as dbg_set_engine
|
||||
from backend.api.pipeline_control import router as pipeline_router, set_engine as pc_set_engine
|
||||
from backend.api.nodes import router as nodes_router
|
||||
from backend.api.metrics import router as metrics_router
|
||||
from backend.api.cause import router as cause_router
|
||||
|
||||
# Other deps
|
||||
from backend.events.schema import EventType
|
||||
from backend.analysis.timeline_store import STORE as TIMELINE_STORE
|
||||
from backend.analysis.failure_propagation import FailurePropagator
|
||||
from backend.config import CONFIG
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# App
|
||||
# ----------------------------------------------------------------
|
||||
ROOT = Path(__file__).resolve().parent.parent # /home/z/my-project/fester
|
||||
|
||||
app = FastAPI(
|
||||
title="Fester",
|
||||
description="Distributed build + scheduler + observability system",
|
||||
version="0.2.0",
|
||||
)
|
||||
|
||||
# Attach WS stream to bus
|
||||
ws_stream = WebSocketStream()
|
||||
attach_ws_stream(ws_stream)
|
||||
|
||||
# Wire a timeline store subscriber so every bus event lands in the journal
|
||||
def _timeline_subscriber(event: Dict[str, Any]):
|
||||
TIMELINE_STORE.append(event)
|
||||
|
||||
bus.subscribe(_timeline_subscriber)
|
||||
|
||||
# Wire a failure propagator so we can compute blast radius later
|
||||
_propagator = FailurePropagator()
|
||||
bus.subscribe(_propagator.ingest_event)
|
||||
|
||||
# Wire metrics counters so /metrics reflects real activity
|
||||
from backend.api.metrics import record_pipeline_action, record_cache_hit
|
||||
|
||||
def _metrics_subscriber(event: Dict[str, Any]):
|
||||
"""Increment counters for task_update + cache_update events."""
|
||||
etype = event.get("type")
|
||||
if etype == "task_update":
|
||||
state = event.get("state")
|
||||
if state:
|
||||
record_pipeline_action(state)
|
||||
elif etype == "cache_update":
|
||||
# Cache hit: try meta.target first, then top-level target, then "unknown"
|
||||
target = event.get("target") or (event.get("meta", {}) or {}).get("target") or "unknown"
|
||||
if event.get("state") == "hit":
|
||||
record_cache_hit(target)
|
||||
|
||||
bus.subscribe(_metrics_subscriber)
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Build history (in-memory; replace with DB later)
|
||||
# ----------------------------------------------------------------
|
||||
# Build history lives in build_runner (defined below).
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Static UI (mount LAST so it doesn't shadow API routes)
|
||||
# ----------------------------------------------------------------
|
||||
@app.get("/")
|
||||
@app.get("/index.html")
|
||||
async def index():
|
||||
return FileResponse(ROOT / "index.html")
|
||||
|
||||
|
||||
@app.get("/style.css")
|
||||
async def style_css():
|
||||
return FileResponse(ROOT / "style.css", media_type="text/css")
|
||||
|
||||
|
||||
@app.get("/ui/app.js")
|
||||
async def app_js():
|
||||
return FileResponse(ROOT / "ui" / "app.js", media_type="application/javascript")
|
||||
|
||||
|
||||
@app.get("/ui/{name}.html")
|
||||
async def ui_page(name: str):
|
||||
p = ROOT / "ui" / f"{name}.html"
|
||||
if not p.exists() or not p.is_file():
|
||||
raise HTTPException(404)
|
||||
return FileResponse(p, media_type="text/html")
|
||||
|
||||
|
||||
# Mount cockpit folder (for the cockpit module)
|
||||
app.mount("/cockpit", StaticFiles(directory=str(ROOT / "cockpit")), name="cockpit")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Include API routers
|
||||
# ----------------------------------------------------------------
|
||||
app.include_router(replay_router)
|
||||
app.include_router(autopsy_router)
|
||||
app.include_router(timeline_router)
|
||||
app.include_router(debugger_router)
|
||||
app.include_router(pipeline_router)
|
||||
app.include_router(nodes_router)
|
||||
app.include_router(metrics_router)
|
||||
app.include_router(cause_router)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Build runner — real engine + synthetic fallback
|
||||
# ----------------------------------------------------------------
|
||||
from backend.pipeline.runner import BuildRunner
|
||||
|
||||
# Nodes for the runner come from CONFIG; the registry is the singleton.
|
||||
_runner_nodes = [{"name": n["name"], "host": n.get("host"), "max_jobs": n.get("max_jobs", 8),
|
||||
"state": "online"} for n in CONFIG.get("nodes", [])]
|
||||
build_runner = BuildRunner(bus, registry, _runner_nodes)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Top-level endpoints that don't fit a router
|
||||
# ----------------------------------------------------------------
|
||||
class BuildBody(BaseModel):
|
||||
cmd: str = "make -j$(nproc)"
|
||||
dir: str = "/home/user/linux"
|
||||
project: str | None = None
|
||||
arch: str = "x86_64"
|
||||
target: str = "linux-gnu"
|
||||
toolchain: str = "gcc"
|
||||
|
||||
|
||||
@app.post("/api/build")
|
||||
async def api_build(body: BuildBody):
|
||||
"""Kick off a build. Returns a build_id immediately; events stream via /ws."""
|
||||
build_id = await build_runner.start(body)
|
||||
return {"status": "started", "build_id": build_id}
|
||||
|
||||
|
||||
@app.get("/api/builds")
|
||||
async def api_builds():
|
||||
"""Build history."""
|
||||
return {"builds": build_runner.list_builds()}
|
||||
|
||||
|
||||
@app.get("/api/builds/{build_id}")
|
||||
async def api_build_detail(build_id: str):
|
||||
state = build_runner.state(build_id)
|
||||
if state:
|
||||
return state
|
||||
raise HTTPException(404, "build not found")
|
||||
|
||||
|
||||
@app.post("/api/builds/{build_id}/cancel")
|
||||
async def api_build_cancel(build_id: str):
|
||||
if build_runner.cancel(build_id):
|
||||
return {"status": "cancelled", "build_id": build_id}
|
||||
raise HTTPException(404, "build not found or not running")
|
||||
|
||||
|
||||
@app.get("/api/sessions")
|
||||
async def api_sessions():
|
||||
"""List replay sessions known to the server (from SessionDB)."""
|
||||
from backend.api.replay import SESSION_DB
|
||||
return {"sessions": SESSION_DB.list_sessions()}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Targets API — backed by backend/targets/catalog.py + ARCHES/RUNTIMES
|
||||
# ----------------------------------------------------------------
|
||||
from backend.targets.catalog import TARGET_SYSTEMS, ARCHES, RUNTIMES
|
||||
|
||||
|
||||
def _build_targets_catalog():
|
||||
"""Combine catalog target systems + arches into a unified toggle list."""
|
||||
out = []
|
||||
# Per-arch cross-compile targets (always present)
|
||||
arch_defaults = {
|
||||
"x86_64": True, # most common, default-on
|
||||
"aarch64": True,
|
||||
"riscv64": False,
|
||||
"arm64": True,
|
||||
}
|
||||
for arch in ARCHES:
|
||||
out.append({
|
||||
"name": f"{arch}-linux-gnu",
|
||||
"enabled": arch_defaults.get(arch, False),
|
||||
"arch": arch,
|
||||
"type": "cross-compile",
|
||||
"source": "arch",
|
||||
})
|
||||
# Per-target-system entries (Gentoo, Buildroot, OpenWrt, etc.)
|
||||
for tname, tinfo in TARGET_SYSTEMS.items():
|
||||
out.append({
|
||||
"name": tname,
|
||||
"enabled": False, # opt-in
|
||||
"type": tinfo.get("type", "unknown"),
|
||||
"toolchain": tinfo.get("toolchain"),
|
||||
"supports_cross": tinfo.get("supports_cross", False),
|
||||
"source": "catalog",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
TARGETS_CATALOG = _build_targets_catalog()
|
||||
|
||||
|
||||
@app.get("/api/targets")
|
||||
async def api_targets():
|
||||
"""List all known build targets (arches + catalog systems)."""
|
||||
return TARGETS_CATALOG
|
||||
|
||||
|
||||
@app.get("/api/targets/arches")
|
||||
async def api_targets_arches():
|
||||
"""List supported architectures."""
|
||||
return {"arches": ARCHES}
|
||||
|
||||
|
||||
@app.get("/api/targets/runtimes")
|
||||
async def api_targets_runtimes():
|
||||
"""List supported execution runtimes."""
|
||||
return {"runtimes": RUNTIMES}
|
||||
|
||||
|
||||
@app.post("/api/targets/{name}")
|
||||
async def api_toggle_target(name: str, body: Dict[str, Any] = Body(default={})):
|
||||
for t in TARGETS_CATALOG:
|
||||
if t["name"] == name:
|
||||
t["enabled"] = bool(body.get("enabled", False))
|
||||
# Broadcast to ws-targets subscribers
|
||||
await _broadcast_targets({
|
||||
"type": "target-update",
|
||||
"data": {"name": name, "enabled": t["enabled"]},
|
||||
})
|
||||
return {"status": "ok", "name": name, "enabled": t["enabled"]}
|
||||
raise HTTPException(404, "target not found")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Release API
|
||||
# ----------------------------------------------------------------
|
||||
class ReleaseBody(BaseModel):
|
||||
repo: str = "https://forgejo.local/project.git"
|
||||
project: str | None = None
|
||||
target: str = "linux-gnu"
|
||||
|
||||
|
||||
@app.post("/api/release")
|
||||
async def api_release(body: ReleaseBody):
|
||||
"""Kick off a release build. Mirrors /api/build but tags the build_id
|
||||
with a 'release-' prefix and uses a longer release-shaped pipeline."""
|
||||
release_id = str(uuid.uuid4())[:8]
|
||||
build_id = f"release-{release_id}"
|
||||
|
||||
# Construct a release-shaped BuildBody and dispatch through BuildRunner
|
||||
release_body = BuildBody(
|
||||
cmd=f"make release REPO={body.repo}",
|
||||
dir="/home/user/release",
|
||||
project=body.project or body.repo,
|
||||
arch="x86_64",
|
||||
target=body.target,
|
||||
toolchain="gcc",
|
||||
)
|
||||
actual_id = await build_runner.start(release_body)
|
||||
|
||||
return {
|
||||
"status": "started",
|
||||
"repo": body.repo,
|
||||
"release_id": release_id,
|
||||
"build_id": actual_id,
|
||||
"pipeline": ["fetch_source", "configure", "build_kernel", "build_modules",
|
||||
"package", "release"],
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Policy API — real implementation backed by an in-memory policy store
|
||||
# ----------------------------------------------------------------
|
||||
from backend.policy.engine import PolicyEngine
|
||||
from backend.api.nodes import PolicyBody as NodePolicyBody # reuse for top-level policy too
|
||||
|
||||
# Singleton policy engine (no DB learning for now — just key/value overrides)
|
||||
POLICY_ENGINE = PolicyEngine(db=None)
|
||||
_POLICY_STORE: Dict[str, Any] = {} # simple key→value overrides
|
||||
|
||||
|
||||
# Pydantic model for /api/policy/set
|
||||
class PolicyBody(BaseModel):
|
||||
key: str
|
||||
value: Any
|
||||
|
||||
|
||||
@app.post("/api/policy/set")
|
||||
async def api_policy_set(body: PolicyBody):
|
||||
"""Set a policy key=value. Currently understood keys:
|
||||
- scheduler.mode (unified|weighted-thermal-aware|cache-first|target-isolated|experimental-intelligence)
|
||||
- scheduler.thermal_cap (float, 0..1)
|
||||
- scheduler.cache_first (bool)
|
||||
- build.default_arch (string)
|
||||
Other keys are stored but not yet acted upon.
|
||||
"""
|
||||
_POLICY_STORE[body.key] = body.value
|
||||
bus.emit(EventType.PIPELINE_UPDATE, state="policy",
|
||||
meta={"key": body.key, "value": body.value, "action": "set"})
|
||||
return {"status": "ok", "key": body.key, "value": body.value,
|
||||
"stored_keys": list(_POLICY_STORE.keys())}
|
||||
|
||||
|
||||
@app.post("/api/policy/clear")
|
||||
async def api_policy_clear(body: Dict[str, Any] = Body(default={})):
|
||||
key = body.get("key")
|
||||
if key and key in _POLICY_STORE:
|
||||
del _POLICY_STORE[key]
|
||||
bus.emit(EventType.PIPELINE_UPDATE, state="policy",
|
||||
meta={"key": key, "action": "clear"})
|
||||
return {"status": "cleared", "key": key, "remaining_keys": list(_POLICY_STORE.keys())}
|
||||
|
||||
|
||||
@app.get("/api/policy")
|
||||
async def api_policy_list():
|
||||
"""List all current policy overrides."""
|
||||
return {"policies": _POLICY_STORE}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Failure propagation (blast radius)
|
||||
# ----------------------------------------------------------------
|
||||
@app.get("/api/propagation/{action}")
|
||||
async def api_propagation(action: str):
|
||||
"""Compute the downstream blast radius if `action` fails."""
|
||||
return _propagator.impact_report(action)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Sessions attach/detach (for the CLI / IDE feature)
|
||||
# ----------------------------------------------------------------
|
||||
@app.post("/api/session/attach")
|
||||
async def api_session_attach(body: Dict[str, Any] = Body(default={})):
|
||||
return {"status": "attached", "action": body.get("action")}
|
||||
|
||||
|
||||
@app.post("/api/session/detach")
|
||||
async def api_session_detach(body: Dict[str, Any] = Body(default={})):
|
||||
return {"status": "detached", "action": body.get("action")}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# WebSocket endpoints
|
||||
# ----------------------------------------------------------------
|
||||
class WSHub:
|
||||
def __init__(self):
|
||||
self.clients: List[WebSocket] = []
|
||||
self.targets_clients: List[WebSocket] = []
|
||||
self.debugger_clients: List[WebSocket] = []
|
||||
|
||||
async def accept(self, ws: WebSocket, kind: str = "main"):
|
||||
await ws.accept()
|
||||
if kind == "main":
|
||||
self.clients.append(ws)
|
||||
elif kind == "targets":
|
||||
self.targets_clients.append(ws)
|
||||
elif kind == "debugger":
|
||||
self.debugger_clients.append(ws)
|
||||
|
||||
|
||||
hub = WSHub()
|
||||
|
||||
# Wire ws_stream (singleton) to broadcast to /ws clients.
|
||||
# We can't await inside the sync bus.emit, so we schedule sends on the
|
||||
# running loop. If there's no running loop (called from sync context),
|
||||
# we silently drop — the WS hub is async-only.
|
||||
def _ws_broadcast(event: Dict[str, Any]):
|
||||
payload = json.dumps(event)
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return # no running loop; skip
|
||||
for c in list(hub.clients):
|
||||
try:
|
||||
loop.create_task(c.send_text(payload))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ws_stream.clients = hub.clients # share the list
|
||||
bus.subscribe(_ws_broadcast)
|
||||
|
||||
|
||||
async def _broadcast_targets(event: Dict[str, Any]):
|
||||
payload = json.dumps(event)
|
||||
dead = []
|
||||
for c in hub.targets_clients:
|
||||
try:
|
||||
await c.send_text(payload)
|
||||
except Exception:
|
||||
dead.append(c)
|
||||
for d in dead:
|
||||
if d in hub.targets_clients:
|
||||
hub.targets_clients.remove(d)
|
||||
|
||||
|
||||
@app.websocket("/ws")
|
||||
async def ws_main(ws: WebSocket):
|
||||
await hub.accept(ws, "main")
|
||||
try:
|
||||
await ws.send_text(json.dumps({
|
||||
"type": "session",
|
||||
"timestamp": time.time(),
|
||||
"data": {"action": "subscribe", "id": "live"},
|
||||
}))
|
||||
while True:
|
||||
await ws.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
if ws in hub.clients:
|
||||
hub.clients.remove(ws)
|
||||
|
||||
|
||||
@app.websocket("/ws-targets")
|
||||
async def ws_targets(ws: WebSocket):
|
||||
await hub.accept(ws, "targets")
|
||||
try:
|
||||
while True:
|
||||
await ws.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
if ws in hub.targets_clients:
|
||||
hub.targets_clients.remove(ws)
|
||||
|
||||
|
||||
@app.websocket("/ws-debugger")
|
||||
async def ws_debugger(ws: WebSocket):
|
||||
"""Real debugger WS — responds to step/pause/resume + streams timeline state."""
|
||||
await hub.accept(ws, "debugger")
|
||||
try:
|
||||
# Send initial state on connect
|
||||
eng = build_runner.get_engine()
|
||||
active = eng is not None
|
||||
await ws.send_text(json.dumps({
|
||||
"type": "timeline-update",
|
||||
"data": {
|
||||
"events": _debugger_timeline_snapshot(),
|
||||
"current": _debugger_current(),
|
||||
"engine_attached": active,
|
||||
},
|
||||
}))
|
||||
|
||||
while True:
|
||||
msg = await ws.receive_text()
|
||||
try:
|
||||
cmd = json.loads(msg)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
action = cmd.get("action")
|
||||
eng = build_runner.get_engine()
|
||||
|
||||
if action == "pause":
|
||||
if eng and hasattr(eng, "pause"):
|
||||
eng.pause()
|
||||
from backend.api.debugger import _DEBUG_STATE
|
||||
_DEBUG_STATE["paused"] = True
|
||||
response = {"type": "state-change", "data": {"state": "paused"}}
|
||||
elif action == "resume":
|
||||
if eng and hasattr(eng, "resume"):
|
||||
eng.resume()
|
||||
from backend.api.debugger import _DEBUG_STATE
|
||||
_DEBUG_STATE["paused"] = False
|
||||
response = {"type": "state-change", "data": {"state": "running"}}
|
||||
elif action == "step_forward" or action == "step":
|
||||
if eng and hasattr(eng, "step"):
|
||||
eng.step()
|
||||
from backend.api.debugger import _DEBUG_STATE
|
||||
_DEBUG_STATE["current_step"] += 1
|
||||
response = {"type": "state-change",
|
||||
"data": {"state": "stepped", "step": _DEBUG_STATE["current_step"]}}
|
||||
elif action == "step_back":
|
||||
from backend.api.debugger import _DEBUG_STATE
|
||||
if _DEBUG_STATE["current_step"] > 0:
|
||||
_DEBUG_STATE["current_step"] -= 1
|
||||
response = {"type": "state-change",
|
||||
"data": {"state": "stepped_back", "step": _DEBUG_STATE["current_step"]}}
|
||||
else:
|
||||
response = {"type": "error", "data": {"error": f"unknown action: {action}"}}
|
||||
|
||||
# Augment with the latest timeline state
|
||||
response["timeline"] = _debugger_timeline_snapshot()
|
||||
response["current"] = _debugger_current()
|
||||
await ws.send_text(json.dumps(response))
|
||||
except WebSocketDisconnect:
|
||||
if ws in hub.debugger_clients:
|
||||
hub.debugger_clients.remove(ws)
|
||||
|
||||
|
||||
def _debugger_timeline_snapshot():
|
||||
"""Return a list of {name, completed, node, state} for the debugger UI."""
|
||||
events = TIMELINE_STORE.all()
|
||||
seen = {}
|
||||
for e in events:
|
||||
action = e.get("action")
|
||||
if not action:
|
||||
continue
|
||||
d = e
|
||||
state = d.get("state")
|
||||
seen[action] = {
|
||||
"name": action,
|
||||
"completed": state == "done",
|
||||
"state": state,
|
||||
"node": d.get("node"),
|
||||
"failed": state == "failed",
|
||||
}
|
||||
return list(seen.values())
|
||||
|
||||
|
||||
def _debugger_current():
|
||||
"""Return the most recent running or scheduled action."""
|
||||
events = TIMELINE_STORE.all()
|
||||
for e in reversed(events):
|
||||
if e.get("state") in ("running", "scheduled"):
|
||||
return {"action": e.get("action"), "state": e.get("state"),
|
||||
"node": e.get("node")}
|
||||
return None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Health
|
||||
# ----------------------------------------------------------------
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {
|
||||
"status": "ok",
|
||||
"version": "0.2.0",
|
||||
"bus_subscribers": len(bus.subscribers),
|
||||
"ws_clients": len(hub.clients),
|
||||
"builds_known": len(build_runner.list_builds()) if build_runner else 0,
|
||||
"timeline_events": TIMELINE_STORE.stats()["total_events"],
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Storage API — btrfs CAS, tmpfs, qcow2 snapshots
|
||||
# ----------------------------------------------------------------
|
||||
from backend.storage.router import (
|
||||
prepare_workspace, maybe_freeze_workspace, storage_status,
|
||||
update_storage_config,
|
||||
)
|
||||
from backend.storage.btrfs_cas import is_btrfs, store_reflink, reflink_snapshot, cas_store, cas_exists
|
||||
from backend.storage.qcow2_freeze import freeze_to_qcow2, qcow2_info, list_snapshots
|
||||
|
||||
|
||||
@app.get("/api/storage/status")
|
||||
async def api_storage_status():
|
||||
"""Return current storage configuration + capability detection."""
|
||||
return storage_status()
|
||||
|
||||
|
||||
@app.post("/api/storage/config")
|
||||
async def api_storage_config_update(body: Dict[str, Any] = Body(default={})):
|
||||
"""Update storage configuration (e.g. {btrfs: true, qcow2_freeze: true})."""
|
||||
return update_storage_config(body)
|
||||
|
||||
|
||||
@app.post("/api/storage/freeze")
|
||||
async def api_storage_freeze(body: Dict[str, Any] = Body(default={})):
|
||||
"""Freeze a workdir into a qcow2 snapshot.
|
||||
|
||||
Body: {"workdir": "/path/to/workdir", "project": "name"}"""
|
||||
workdir = body.get("workdir")
|
||||
project = body.get("project", "manual")
|
||||
if not workdir:
|
||||
raise HTTPException(400, "workdir required")
|
||||
result = maybe_freeze_workspace(workdir, project)
|
||||
if result is None:
|
||||
return {"status": "disabled", "reason": "qcow2_freeze not enabled in config"}
|
||||
return {"status": "frozen", "path": result, "project": project}
|
||||
|
||||
|
||||
@app.get("/api/storage/snapshots")
|
||||
async def api_storage_snapshots():
|
||||
"""List all qcow2 snapshots."""
|
||||
from backend.storage.config import load_storage_config
|
||||
cfg = load_storage_config()
|
||||
return {"snapshots": list_snapshots(cfg.get("qcow2_path", "/var/lib/fester/snapshots"))}
|
||||
|
||||
|
||||
@app.post("/api/storage/reflink")
|
||||
async def api_storage_reflink(body: Dict[str, Any] = Body(default={})):
|
||||
"""Reflink-copy a file or directory (CoW snapshot on btrfs).
|
||||
|
||||
Body: {"src": "/path/to/src", "dst": "/path/to/dst"}"""
|
||||
src = body.get("src")
|
||||
dst = body.get("dst")
|
||||
if not src or not dst:
|
||||
raise HTTPException(400, "src and dst required")
|
||||
if not Path(src).exists():
|
||||
raise HTTPException(404, f"src not found: {src}")
|
||||
|
||||
from pathlib import Path as _P
|
||||
if _P(src).is_dir():
|
||||
stats = reflink_snapshot(src, dst)
|
||||
return {"status": "ok", "mode": "directory", "stats": stats}
|
||||
else:
|
||||
reflinked = store_reflink(src, dst)
|
||||
return {"status": "ok", "mode": "file", "reflinked": reflinked}
|
||||
|
||||
|
||||
@app.post("/api/workspace/prepare")
|
||||
async def api_workspace_prepare(body: Dict[str, Any] = Body(default={})):
|
||||
"""Prepare a workspace for a build (tmpfs + optional btrfs CAS).
|
||||
|
||||
Body: {"project": "name"}"""
|
||||
project = body.get("project", "default")
|
||||
return prepare_workspace(project)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Actions API — tmux session inspection + node shell attach
|
||||
# ----------------------------------------------------------------
|
||||
from backend.integrations.tmux import TmuxManager as _TmuxMgr
|
||||
from backend.integrations.mosh import MoshManager as _MoshMgr
|
||||
|
||||
_tmux_mgr = _TmuxMgr()
|
||||
_mosh_mgr = _MoshMgr()
|
||||
|
||||
|
||||
@app.get("/api/actions/active")
|
||||
async def api_actions_active():
|
||||
"""List all active fester-* tmux sessions (running actions)."""
|
||||
if not _tmux_mgr.available:
|
||||
return {"available": False, "sessions": [], "reason": "tmux not installed"}
|
||||
return {
|
||||
"available": True,
|
||||
"sessions": _tmux_mgr.list_fester_sessions(),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/actions/{name}/tmux")
|
||||
async def api_action_tmux_state(name: str):
|
||||
"""Get state of the tmux session for a specific action."""
|
||||
if not _tmux_mgr.available:
|
||||
return {"available": False, "reason": "tmux not installed"}
|
||||
# Try with and without build_id suffix
|
||||
candidates = [f"fester-{name}"]
|
||||
# Also try matching any session that starts with fester-{name}-
|
||||
sessions = _tmux_mgr.list_fester_sessions()
|
||||
for s in sessions:
|
||||
if s["name"].startswith(f"fester-{name}-"):
|
||||
candidates.append(s["name"])
|
||||
for cand in candidates:
|
||||
state = _tmux_mgr.session_state(cand)
|
||||
if state:
|
||||
output = _tmux_mgr.capture_output(cand, lines=200) or ""
|
||||
return {
|
||||
"available": True,
|
||||
"name": cand,
|
||||
"state": state,
|
||||
"output_tail": output[-2000:] if output else None,
|
||||
}
|
||||
return {"available": True, "exists": False, "name": name}
|
||||
|
||||
|
||||
@app.get("/api/actions/{name}/tmux/output")
|
||||
async def api_action_tmux_output(name: str, lines: int = 1000):
|
||||
"""Capture the last N lines of tmux pane output for an action."""
|
||||
if not _tmux_mgr.available:
|
||||
return {"available": False, "reason": "tmux not installed"}
|
||||
# Find the session
|
||||
candidates = [f"fester-{name}"]
|
||||
sessions = _tmux_mgr.list_fester_sessions()
|
||||
for s in sessions:
|
||||
if s["name"].startswith(f"fester-{name}-"):
|
||||
candidates.append(s["name"])
|
||||
for cand in candidates:
|
||||
if _tmux_mgr.session_exists(cand):
|
||||
output = _tmux_mgr.capture_output(cand, lines=lines) or ""
|
||||
return {
|
||||
"available": True,
|
||||
"name": cand,
|
||||
"lines": len(output.split("\n")) if output else 0,
|
||||
"output": output,
|
||||
}
|
||||
return {"available": True, "exists": False, "name": name}
|
||||
|
||||
|
||||
@app.post("/api/actions/{name}/tmux/kill")
|
||||
async def api_action_tmux_kill(name: str):
|
||||
"""Kill the tmux session for an action."""
|
||||
if not _tmux_mgr.available:
|
||||
return {"available": False, "reason": "tmux not installed"}
|
||||
candidates = [f"fester-{name}"]
|
||||
sessions = _tmux_mgr.list_fester_sessions()
|
||||
for s in sessions:
|
||||
if s["name"].startswith(f"fester-{name}-"):
|
||||
candidates.append(s["name"])
|
||||
killed = []
|
||||
for cand in candidates:
|
||||
if _tmux_mgr.kill_session(cand):
|
||||
killed.append(cand)
|
||||
return {"killed": killed, "count": len(killed)}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Node shell — mosh / ssh command builder
|
||||
# ----------------------------------------------------------------
|
||||
class ShellBody(BaseModel):
|
||||
user: Optional[str] = None
|
||||
port: Optional[int] = None
|
||||
ssh_key: Optional[str] = None
|
||||
method: str = "mosh" # mosh | ssh
|
||||
|
||||
|
||||
@app.get("/api/nodes/{name}/shell")
|
||||
async def api_node_shell(name: str):
|
||||
"""Return the command string to attach a shell to a node (mosh or ssh).
|
||||
Doesn't actually exec it — the caller's terminal does."""
|
||||
# Find the node
|
||||
cfg = None
|
||||
for n in CONFIG.get("nodes", []):
|
||||
if n["name"] == name:
|
||||
cfg = n
|
||||
break
|
||||
if not cfg:
|
||||
raise HTTPException(404, f"node {name} not configured")
|
||||
host = cfg.get("host")
|
||||
if not host:
|
||||
raise HTTPException(400, f"node {name} has no host")
|
||||
|
||||
return {
|
||||
"node": name,
|
||||
"host": host,
|
||||
"mosh_available": _mosh_mgr.available,
|
||||
"ssh_available": shutil.which("ssh") is not None,
|
||||
"commands": {
|
||||
"mosh": f"mosh {host}",
|
||||
"ssh": f"ssh {host}",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/nodes/{name}/shell")
|
||||
async def api_node_shell_custom(name: str, body: ShellBody):
|
||||
"""Build a custom shell command for a node with user/port/ssh-key options."""
|
||||
cfg = None
|
||||
for n in CONFIG.get("nodes", []):
|
||||
if n["name"] == name:
|
||||
cfg = n
|
||||
break
|
||||
if not cfg:
|
||||
raise HTTPException(404, f"node {name} not configured")
|
||||
host = cfg.get("host")
|
||||
if not host:
|
||||
raise HTTPException(400, f"node {name} has no host")
|
||||
|
||||
target = f"{body.user}@{host}" if body.user else host
|
||||
cmd = _mosh_mgr.build_command(target, port=body.port, ssh_key=body.ssh_key,
|
||||
method=body.method)
|
||||
return {
|
||||
"node": name,
|
||||
"host": host,
|
||||
"method": body.method,
|
||||
"command": cmd,
|
||||
"copy_to_clipboard": cmd,
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Ambient loop — probe real node agents, fall back to drift if unreachable.
|
||||
# Set FESTER_NO_DRIFT=1 to disable synthetic drift entirely (production).
|
||||
# ----------------------------------------------------------------
|
||||
async def ambient_loop():
|
||||
"""Probe each configured node's agent every 4s. If the agent responds,
|
||||
update the registry with real metrics. If not, optionally drift the
|
||||
last-known values so the UI doesn't go stale."""
|
||||
import os
|
||||
import random
|
||||
from backend.nodes.probe import probe_async, normalize
|
||||
from backend.storage.sqlite_db import STORAGE
|
||||
|
||||
NO_DRIFT = os.environ.get("FESTER_NO_DRIFT") == "1"
|
||||
await asyncio.sleep(2)
|
||||
|
||||
while True:
|
||||
for n in CONFIG.get("nodes", []):
|
||||
name = n["name"]
|
||||
host = n.get("host")
|
||||
|
||||
# Try real probe first
|
||||
raw = await probe_async(host) if host else None
|
||||
|
||||
if raw is not None:
|
||||
# Real agent responded — use its data
|
||||
state = normalize(raw, name)
|
||||
registry.update(
|
||||
name,
|
||||
cpu_load=state["cpu_load"],
|
||||
memory_load=state["memory_load"],
|
||||
temp=state["temp"],
|
||||
instability=state["instability"],
|
||||
active_jobs=state["active_jobs"],
|
||||
last_seen=state["last_seen"],
|
||||
)
|
||||
cur = registry.get(name)
|
||||
# Persist node state to SQLite
|
||||
if STORAGE:
|
||||
try:
|
||||
STORAGE.save_node_state(name, {
|
||||
"state": state.get("state", "online"),
|
||||
"cpu_load": cur.cpu_load,
|
||||
"memory_load": cur.memory_load,
|
||||
"temp": cur.temp,
|
||||
"instability": cur.instability,
|
||||
"active_jobs": cur.active_jobs,
|
||||
"max_jobs": state.get("max_jobs", n.get("max_jobs", 8)),
|
||||
"last_seen": cur.last_seen,
|
||||
"labels": state.get("labels", {}),
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
# Emit with state from probe (may be "online" or "degraded")
|
||||
bus.emit(
|
||||
EventType.NODE_UPDATE,
|
||||
node=name,
|
||||
state=state.get("state", "online"),
|
||||
meta={
|
||||
"heat": cur.temp,
|
||||
"jobs": cur.active_jobs,
|
||||
"cpu_load": cur.cpu_load,
|
||||
"instability": cur.instability,
|
||||
"source": "agent",
|
||||
},
|
||||
)
|
||||
elif not NO_DRIFT:
|
||||
# Agent unreachable — drift the last-known values
|
||||
existing = registry.get(name)
|
||||
if existing is None:
|
||||
# First-seen: initialize with plausible values
|
||||
registry.update(
|
||||
name,
|
||||
cpu_load=random.uniform(20, 70),
|
||||
memory_load=random.uniform(30, 60),
|
||||
temp=random.uniform(40, 70),
|
||||
instability=random.uniform(0, 0.15),
|
||||
active_jobs=random.randint(0, max(1, n.get("max_jobs", 8) // 4)),
|
||||
last_seen=time.time(),
|
||||
)
|
||||
else:
|
||||
registry.update(
|
||||
name,
|
||||
cpu_load=max(0, min(100, existing.cpu_load + random.uniform(-5, 5))),
|
||||
memory_load=max(0, min(100, existing.memory_load + random.uniform(-2, 2))),
|
||||
temp=max(20, min(95, existing.temp + random.uniform(-3, 3))),
|
||||
instability=max(0, min(1, existing.instability + random.uniform(-0.05, 0.05))),
|
||||
active_jobs=max(0, min(n.get("max_jobs", 8), existing.active_jobs + random.randint(-1, 1))),
|
||||
last_seen=time.time(),
|
||||
)
|
||||
|
||||
cur = registry.get(name)
|
||||
# Persist drifted state too
|
||||
if STORAGE:
|
||||
try:
|
||||
STORAGE.save_node_state(name, {
|
||||
"state": "online",
|
||||
"cpu_load": cur.cpu_load,
|
||||
"memory_load": cur.memory_load,
|
||||
"temp": cur.temp,
|
||||
"instability": cur.instability,
|
||||
"active_jobs": cur.active_jobs,
|
||||
"max_jobs": n.get("max_jobs", 8),
|
||||
"last_seen": cur.last_seen,
|
||||
"labels": {},
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
bus.emit(
|
||||
EventType.NODE_UPDATE,
|
||||
node=name,
|
||||
state="online",
|
||||
meta={
|
||||
"heat": cur.temp,
|
||||
"jobs": cur.active_jobs,
|
||||
"cpu_load": cur.cpu_load,
|
||||
"instability": cur.instability,
|
||||
"source": "drift",
|
||||
},
|
||||
)
|
||||
# If NO_DRIFT and no agent, we simply skip — node stays offline
|
||||
|
||||
await asyncio.sleep(4)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def on_startup():
|
||||
asyncio.create_task(ambient_loop())
|
||||
# Optional: kick off a build every 60s for demo purposes.
|
||||
# Disabled by default — set FESTER_AUTOBUILD=1 to enable.
|
||||
import os
|
||||
if os.environ.get("FESTER_AUTOBUILD") == "1":
|
||||
async def looper():
|
||||
await asyncio.sleep(15) # let things warm up first
|
||||
while True:
|
||||
await build_runner.start(
|
||||
BuildBody(cmd="make -j$(nproc)", dir="/home/user/linux"),
|
||||
)
|
||||
await asyncio.sleep(60)
|
||||
asyncio.create_task(looper())
|
||||
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# CLI entrypoint
|
||||
# ----------------------------------------------------------------
|
||||
def main():
|
||||
import uvicorn
|
||||
uvicorn.run("backend.main:app", host="0.0.0.0", port=8080, reload=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
class ClusterState:
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.nodes = {}
|
||||
|
||||
def register_node(self, node_id, node_state):
|
||||
|
||||
self.nodes[node_id] = node_state
|
||||
|
||||
# -----------------------------
|
||||
# GLOBAL AGGREGATION (VERY CHEAP)
|
||||
# -----------------------------
|
||||
def snapshot(self):
|
||||
|
||||
hot_nodes = 0
|
||||
total_temp = 0
|
||||
total_cpu = 0
|
||||
instability_sum = 0
|
||||
n = len(self.nodes)
|
||||
|
||||
if n == 0:
|
||||
return {}
|
||||
|
||||
for node in self.nodes.values():
|
||||
|
||||
s = node.snapshot()
|
||||
|
||||
total_temp += s["avg_temp"]
|
||||
total_cpu += s["avg_cpu"]
|
||||
instability_sum += s["instability"]
|
||||
|
||||
if s["avg_temp"] > 80:
|
||||
hot_nodes += 1
|
||||
|
||||
return {
|
||||
"avg_cluster_temp": total_temp / n,
|
||||
"avg_cluster_cpu": total_cpu / n,
|
||||
"instability": instability_sum / n,
|
||||
"hot_nodes": hot_nodes
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import time
|
||||
import psutil
|
||||
|
||||
|
||||
class MetricsCollector:
|
||||
"""
|
||||
Lightweight node telemetry sampler.
|
||||
"""
|
||||
|
||||
def sample(self):
|
||||
|
||||
return {
|
||||
"cpu": psutil.cpu_percent(interval=0.1),
|
||||
"memory": psutil.virtual_memory().percent,
|
||||
"load": psutil.getloadavg()[0] if hasattr(psutil, "getloadavg") else 0,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
from backend.metrics.target_scoring import TargetScorer
|
||||
from backend.metrics.target_labeler import TargetLabeler
|
||||
from backend.metrics.profile_store import save
|
||||
|
||||
|
||||
class LiveTargetState:
|
||||
|
||||
def __init__(self, target_key):
|
||||
self.target = target_key
|
||||
|
||||
# -----------------------------
|
||||
# RUNNING STATE (incremental)
|
||||
# -----------------------------
|
||||
self.cpu_sum = 0
|
||||
self.temp_sum = 0
|
||||
self.samples = 0
|
||||
|
||||
self.peak_temp = 0
|
||||
|
||||
self.actions = 0
|
||||
self.cache_hits = 0
|
||||
self.failures = 0
|
||||
|
||||
# -----------------------------
|
||||
# CACHED DERIVED VALUES
|
||||
# -----------------------------
|
||||
self.cached_score = None
|
||||
self.cached_labels = None
|
||||
self.dirty = True
|
||||
|
||||
|
||||
class FesterIntelligence:
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.states = {}
|
||||
self.scorer = TargetScorer()
|
||||
self.labeler = TargetLabeler()
|
||||
|
||||
# -----------------------------
|
||||
# INIT
|
||||
# -----------------------------
|
||||
def start_target(self, target_key):
|
||||
|
||||
self.states[target_key] = LiveTargetState(target_key)
|
||||
|
||||
# -----------------------------
|
||||
# FAST UPDATES (O(1))
|
||||
# -----------------------------
|
||||
def sample(self, target_key, node):
|
||||
|
||||
s = self.states[target_key]
|
||||
|
||||
cpu = node.get("cpu_load", 0)
|
||||
temp = node.get("temp", 0)
|
||||
|
||||
s.cpu_sum += cpu
|
||||
s.temp_sum += temp
|
||||
s.samples += 1
|
||||
|
||||
if temp > s.peak_temp:
|
||||
s.peak_temp = temp
|
||||
|
||||
s.dirty = True
|
||||
|
||||
def record_event(self, target_key, event):
|
||||
|
||||
s = self.states[target_key]
|
||||
|
||||
if event == "action":
|
||||
s.actions += 1
|
||||
|
||||
elif event == "cache_hit":
|
||||
s.cache_hits += 1
|
||||
|
||||
elif event == "failure":
|
||||
s.failures += 1
|
||||
|
||||
s.dirty = True
|
||||
|
||||
# -----------------------------
|
||||
# FAST SNAPSHOT (NO RECOMPUTE UNLESS DIRTY)
|
||||
# -----------------------------
|
||||
def snapshot(self, target_key):
|
||||
|
||||
s = self.states.get(target_key)
|
||||
|
||||
if not s:
|
||||
return None
|
||||
|
||||
# -----------------------------
|
||||
# RETURN CACHED RESULT
|
||||
# -----------------------------
|
||||
if not s.dirty and s.cached_score is not None:
|
||||
return {
|
||||
"score": s.cached_score,
|
||||
"labels": s.cached_labels
|
||||
}
|
||||
|
||||
# -----------------------------
|
||||
# COMPUTE AGGREGATES
|
||||
# -----------------------------
|
||||
avg_cpu = s.cpu_sum / s.samples if s.samples else 0
|
||||
avg_temp = s.temp_sum / s.samples if s.samples else 0
|
||||
|
||||
cache_ratio = s.cache_hits / s.actions if s.actions else 0
|
||||
|
||||
profile = {
|
||||
"duration": 0, # intentionally not scheduler-relevant anymore
|
||||
"avg_cpu": avg_cpu,
|
||||
"avg_temp": avg_temp,
|
||||
"peak_temp": s.peak_temp,
|
||||
"cache_ratio": cache_ratio,
|
||||
"failures": s.failures
|
||||
}
|
||||
|
||||
score = self.scorer.score(profile)
|
||||
labels = self.labeler.label(score)
|
||||
|
||||
# -----------------------------
|
||||
# CACHE RESULT
|
||||
# -----------------------------
|
||||
s.cached_score = score
|
||||
s.cached_labels = labels
|
||||
s.dirty = False
|
||||
|
||||
return {
|
||||
"score": score,
|
||||
"labels": labels
|
||||
}
|
||||
|
||||
# -----------------------------
|
||||
# FINALIZE (ARCHIVAL ONLY)
|
||||
# -----------------------------
|
||||
def finalize(self, target_key):
|
||||
|
||||
s = self.states[target_key]
|
||||
|
||||
snapshot = self.snapshot(target_key)
|
||||
|
||||
save({
|
||||
"target": target_key,
|
||||
"score": snapshot["score"],
|
||||
"labels": snapshot["labels"]
|
||||
})
|
||||
|
||||
return snapshot
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
from dataclasses import dataclass, field
|
||||
from typing import Dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeState:
|
||||
name: str
|
||||
|
||||
cpu_load: float = 0.0
|
||||
memory_load: float = 0.0
|
||||
temp: float = 0.0
|
||||
instability: float = 0.0
|
||||
|
||||
active_jobs: int = 0
|
||||
last_seen: float = 0.0
|
||||
|
||||
labels: Dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class NodeStateRegistry:
|
||||
|
||||
def __init__(self):
|
||||
self.nodes = {}
|
||||
|
||||
def update(self, name, **kwargs):
|
||||
|
||||
if name not in self.nodes:
|
||||
self.nodes[name] = NodeState(name=name)
|
||||
|
||||
node = self.nodes[name]
|
||||
|
||||
for k, v in kwargs.items():
|
||||
setattr(node, k, v)
|
||||
|
||||
def get(self, name):
|
||||
return self.nodes.get(name)
|
||||
|
||||
def all(self):
|
||||
return list(self.nodes.values())
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
from backend.events.bus import EventBus
|
||||
from backend.metrics.node_state import NodeStateRegistry
|
||||
|
||||
|
||||
class ObservabilityHub:
|
||||
"""
|
||||
Single fusion layer for:
|
||||
- events (what happened)
|
||||
- metrics (what is happening)
|
||||
- state (what is true now)
|
||||
"""
|
||||
|
||||
def __init__(self, registry: NodeStateRegistry, bus: EventBus):
|
||||
self.registry = registry
|
||||
self.bus = bus
|
||||
self.subscribers = []
|
||||
|
||||
# -----------------------------
|
||||
# STREAM OUT TO UI / PROMETHEUS / GRAFANA
|
||||
# -----------------------------
|
||||
def subscribe(self, fn):
|
||||
self.subscribers.append(fn)
|
||||
|
||||
def emit(self, event: dict):
|
||||
|
||||
# enrich event with live state snapshot
|
||||
if event.get("node"):
|
||||
node = self.registry.get(event["node"])
|
||||
if node:
|
||||
event["node_state"] = {
|
||||
"cpu": node.cpu_load,
|
||||
"temp": node.temp,
|
||||
"instability": node.instability,
|
||||
"active_jobs": node.active_jobs,
|
||||
}
|
||||
|
||||
for sub in self.subscribers:
|
||||
sub(event)
|
||||
|
||||
# -----------------------------
|
||||
# CONNECT EVENT BUS
|
||||
# -----------------------------
|
||||
def attach_bus(self):
|
||||
|
||||
def forward(event):
|
||||
self.emit(event)
|
||||
|
||||
self.bus.subscribe(forward)
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
PROFILES = []
|
||||
|
||||
|
||||
def save(profile):
|
||||
PROFILES.append(profile)
|
||||
|
||||
|
||||
def all():
|
||||
return PROFILES
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
from prometheus_client import Gauge, Counter, start_http_server
|
||||
|
||||
# -----------------------------
|
||||
# METRICS
|
||||
# -----------------------------
|
||||
|
||||
# Node metrics
|
||||
node_cpu = Gauge("fester_node_cpu", "CPU load per node", ["node"])
|
||||
node_temp = Gauge("fester_node_temp", "Temperature per node", ["node"])
|
||||
|
||||
# Pipeline metrics
|
||||
pipeline_actions = Counter("fester_pipeline_actions_total", "Total actions", ["state"])
|
||||
cache_hits = Counter("fester_cache_hits_total", "Cache hits", ["target"])
|
||||
|
||||
# Scheduler metrics
|
||||
scheduler_score = Gauge("fester_scheduler_score", "Node score", ["node", "target"])
|
||||
|
||||
# -----------------------------
|
||||
# EXPORTER START
|
||||
# -----------------------------
|
||||
def start_metrics_server(port=9109):
|
||||
start_http_server(port)
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
class TargetLabeler:
|
||||
|
||||
def label(self, score):
|
||||
|
||||
labels = []
|
||||
|
||||
# -----------------------------
|
||||
# THERMAL
|
||||
# -----------------------------
|
||||
if score["thermal"] > 80:
|
||||
labels.append("HOT")
|
||||
elif score["thermal"] < 60:
|
||||
labels.append("COOL")
|
||||
|
||||
# -----------------------------
|
||||
# COST
|
||||
# -----------------------------
|
||||
if score["cost"] > 100:
|
||||
labels.append("EXPENSIVE")
|
||||
|
||||
# -----------------------------
|
||||
# STABILITY
|
||||
# -----------------------------
|
||||
if score["instability"] > 5:
|
||||
labels.append("UNSTABLE")
|
||||
|
||||
if not labels:
|
||||
labels.append("SAFE")
|
||||
|
||||
return labels
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
import time
|
||||
|
||||
|
||||
class TargetProfile:
|
||||
|
||||
def __init__(self, target_key):
|
||||
|
||||
self.target = target_key
|
||||
|
||||
self.start_time = time.time()
|
||||
|
||||
self.cpu_samples = []
|
||||
self.temp_samples = []
|
||||
|
||||
self.cache_hits = 0
|
||||
self.actions = 0
|
||||
self.failures = 0
|
||||
|
||||
# -----------------------------
|
||||
# RECORD SAMPLE
|
||||
# -----------------------------
|
||||
def sample(self, node):
|
||||
|
||||
self.cpu_samples.append(node.get("cpu_load", 0))
|
||||
self.temp_samples.append(node.get("temp", 0))
|
||||
|
||||
# -----------------------------
|
||||
# RECORD EVENTS
|
||||
# -----------------------------
|
||||
def record_action(self):
|
||||
self.actions += 1
|
||||
|
||||
def record_cache_hit(self):
|
||||
self.cache_hits += 1
|
||||
|
||||
def record_failure(self):
|
||||
self.failures += 1
|
||||
|
||||
# -----------------------------
|
||||
# FINALIZE PROFILE
|
||||
# -----------------------------
|
||||
def finalize(self):
|
||||
|
||||
duration = time.time() - self.start_time
|
||||
|
||||
avg_cpu = sum(self.cpu_samples) / len(self.cpu_samples) if self.cpu_samples else 0
|
||||
avg_temp = sum(self.temp_samples) / len(self.temp_samples) if self.temp_samples else 0
|
||||
peak_temp = max(self.temp_samples) if self.temp_samples else 0
|
||||
|
||||
cache_ratio = self.cache_hits / self.actions if self.actions else 0
|
||||
|
||||
return {
|
||||
"target": self.target,
|
||||
"duration": duration,
|
||||
"avg_cpu": avg_cpu,
|
||||
"avg_temp": avg_temp,
|
||||
"peak_temp": peak_temp,
|
||||
"cache_ratio": cache_ratio,
|
||||
"failures": self.failures
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
class TargetScorer:
|
||||
|
||||
def score(self, profile):
|
||||
|
||||
# -----------------------------
|
||||
# THERMAL RISK SCORE (0–100)
|
||||
# -----------------------------
|
||||
thermal = (
|
||||
profile["peak_temp"] * 0.5 +
|
||||
profile["avg_temp"] * 0.5
|
||||
)
|
||||
|
||||
# -----------------------------
|
||||
# COST SCORE
|
||||
# -----------------------------
|
||||
cost = profile["duration"] * (1 - profile["cache_ratio"] + 0.1)
|
||||
|
||||
# -----------------------------
|
||||
# STABILITY SCORE
|
||||
# -----------------------------
|
||||
instability = profile["failures"] * 10
|
||||
|
||||
return {
|
||||
"thermal": thermal,
|
||||
"cost": cost,
|
||||
"instability": instability
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
"""
|
||||
backend.nodes package.
|
||||
|
||||
Re-exports the canonical NodeState + NodeStateRegistry from
|
||||
state_model.py for callers that do `from backend.nodes import NodeState`.
|
||||
|
||||
The legacy CLI script (formerly backend/nodes.py) is now at
|
||||
backend/nodes_cli.py.
|
||||
"""
|
||||
|
||||
from backend.nodes.state_model import NodeState, NodeStateRegistry
|
||||
from backend.nodes.roles import NodeRole
|
||||
from backend.nodes.roles_store import (
|
||||
load_roles, save_roles, get_role, DEFAULT_ROLE, ROLE_DB,
|
||||
)
|
||||
from backend.nodes.probe import probe_sync, probe_async, probe_node, normalize
|
||||
|
||||
__all__ = [
|
||||
"NodeState",
|
||||
"NodeStateRegistry",
|
||||
"NodeRole",
|
||||
"load_roles",
|
||||
"save_roles",
|
||||
"get_role",
|
||||
"DEFAULT_ROLE",
|
||||
"ROLE_DB",
|
||||
"probe_sync",
|
||||
"probe_async",
|
||||
"probe_node",
|
||||
"normalize",
|
||||
]
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
"""
|
||||
Node probe — query a remote fester-agent for live state.
|
||||
|
||||
Each cluster node is expected to run an HTTP agent on port 8787 (the
|
||||
convention established by backend/nodes_cli.py) that returns JSON like:
|
||||
|
||||
{
|
||||
"state": "online",
|
||||
"load": "0.42 0.51 0.48 2/128 1234",
|
||||
"temp": 62,
|
||||
"jobs": 4,
|
||||
"max_jobs": 24,
|
||||
"memory": {"used": 8.2, "total": 32.0, "unit": "GB"},
|
||||
"labels": {"arch": "x86_64", "role": "compile"}
|
||||
}
|
||||
|
||||
If the agent is unreachable, the probe returns None and the caller can
|
||||
fall back to synthetic drift.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
try:
|
||||
import aiohttp
|
||||
_HAVE_AIOHTTP = True
|
||||
except ImportError:
|
||||
_HAVE_AIOHTTP = False
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
AGENT_PORT = 8787
|
||||
DEFAULT_TIMEOUT = 1.5 # seconds
|
||||
|
||||
|
||||
def probe_sync(host: str, port: int = AGENT_PORT, timeout: float = DEFAULT_TIMEOUT) -> Optional[Dict[str, Any]]:
|
||||
"""Synchronous probe — for use outside the event loop."""
|
||||
if not host:
|
||||
return None
|
||||
url = f"http://{host}:{port}/status"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
if resp.status != 200:
|
||||
return None
|
||||
import json
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, ConnectionError, TimeoutError, OSError):
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def probe_async(host: str, port: int = AGENT_PORT, timeout: float = DEFAULT_TIMEOUT) -> Optional[Dict[str, Any]]:
|
||||
"""Async probe — preferred when running inside FastAPI / asyncio."""
|
||||
if not host:
|
||||
return None
|
||||
url = f"http://{host}:{port}/status"
|
||||
if _HAVE_AIOHTTP:
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status != 200:
|
||||
return None
|
||||
return await resp.json()
|
||||
except (asyncio.TimeoutError, aiohttp.ClientError, OSError):
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
else:
|
||||
# Fallback: run sync probe in a thread
|
||||
return await asyncio.to_thread(probe_sync, host, port, timeout)
|
||||
|
||||
|
||||
def normalize(agent_data: Optional[Dict[str, Any]], node_name: str) -> Dict[str, Any]:
|
||||
"""Convert an agent's response into a normalized NodeState shape.
|
||||
|
||||
If agent_data is None (probe failed), returns a minimal offline shape."""
|
||||
if not agent_data:
|
||||
return {
|
||||
"name": node_name,
|
||||
"state": "offline",
|
||||
"cpu_load": 0.0,
|
||||
"memory_load": 0.0,
|
||||
"temp": 0.0,
|
||||
"instability": 1.0, # mark unstable since we can't reach it
|
||||
"active_jobs": 0,
|
||||
"last_seen": 0.0,
|
||||
}
|
||||
|
||||
# Parse load average string like "0.42 0.51 0.48 2/128 1234"
|
||||
load_str = agent_data.get("load", "0 0 0")
|
||||
try:
|
||||
cpu_load = float(str(load_str).split()[0]) * 100 # 1.0 = 100%
|
||||
except (ValueError, IndexError):
|
||||
cpu_load = 0.0
|
||||
|
||||
# Memory: agent reports {used, total, unit} in GB
|
||||
mem = agent_data.get("memory", {})
|
||||
if isinstance(mem, dict) and mem.get("total"):
|
||||
memory_load = (float(mem.get("used", 0)) / float(mem["total"])) * 100
|
||||
else:
|
||||
memory_load = 0.0
|
||||
|
||||
return {
|
||||
"name": node_name,
|
||||
"state": agent_data.get("state", "online"),
|
||||
"cpu_load": cpu_load,
|
||||
"memory_load": memory_load,
|
||||
"temp": float(agent_data.get("temp", 0)),
|
||||
"instability": float(agent_data.get("instability", 0.0)),
|
||||
"active_jobs": int(agent_data.get("jobs", 0)),
|
||||
"max_jobs": int(agent_data.get("max_jobs", 0)),
|
||||
"labels": agent_data.get("labels", {}),
|
||||
"last_seen": time.time(),
|
||||
}
|
||||
|
||||
|
||||
async def probe_node(name: str, host: str, port: int = AGENT_PORT) -> Dict[str, Any]:
|
||||
"""Probe one node and return normalized state.
|
||||
Always returns a dict (offline if probe failed)."""
|
||||
raw = await probe_async(host, port)
|
||||
return normalize(raw, name)
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeRole:
|
||||
name: str
|
||||
|
||||
# ----------------------------
|
||||
# cache behavior
|
||||
# ----------------------------
|
||||
cache_writer: bool = True # can store artifacts
|
||||
cache_reader: bool = True # can fetch artifacts
|
||||
cache_priority: int = 50 # higher = preferred cache source
|
||||
|
||||
# ----------------------------
|
||||
# compute behavior
|
||||
# ----------------------------
|
||||
compile_weight: float = 1.0 # scheduler weight multiplier
|
||||
distcc_enabled: bool = True
|
||||
|
||||
# ----------------------------
|
||||
# thermal safety
|
||||
# ----------------------------
|
||||
max_thermal_state: float = 0.85
|
||||
|
||||
# ----------------------------
|
||||
# storage capabilities
|
||||
# ----------------------------
|
||||
tmpfs_ok: bool = True
|
||||
btrfs_ok: bool = False
|
||||
qcow2_ok: bool = False
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from backend.nodes.roles import NodeRole
|
||||
|
||||
|
||||
# Where node role overrides are persisted. Allow override via env var;
|
||||
# otherwise default to a user-writable path under /var/lib/fester (or
|
||||
# fall back to a tmp dir if neither is writable).
|
||||
def _resolve_role_db() -> Path:
|
||||
env = os.environ.get("FESTER_ROLE_DB")
|
||||
if env:
|
||||
return Path(env)
|
||||
|
||||
candidates = [
|
||||
Path("/etc/fester/node_roles.json"),
|
||||
Path("/var/lib/fester/node_roles.json"),
|
||||
Path.home() / ".fester" / "node_roles.json",
|
||||
Path("/tmp/fester_node_roles.json"),
|
||||
]
|
||||
for c in candidates:
|
||||
try:
|
||||
c.parent.mkdir(parents=True, exist_ok=True)
|
||||
# touch to verify writability
|
||||
if not c.exists():
|
||||
c.write_text("{}")
|
||||
else:
|
||||
# verify we can write to it
|
||||
with open(c, "a"):
|
||||
pass
|
||||
return c
|
||||
except (OSError, PermissionError):
|
||||
continue
|
||||
# Last resort: in-memory only (caller must handle None)
|
||||
return None
|
||||
|
||||
|
||||
ROLE_DB = _resolve_role_db()
|
||||
|
||||
DEFAULT_ROLE = NodeRole(name="default")
|
||||
|
||||
|
||||
def load_roles():
|
||||
if ROLE_DB is None or not ROLE_DB.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
raw = json.loads(ROLE_DB.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
roles = {}
|
||||
|
||||
for node, cfg in raw.items():
|
||||
try:
|
||||
roles[node] = NodeRole(**cfg)
|
||||
except TypeError:
|
||||
# Schema drift — skip malformed entries
|
||||
continue
|
||||
|
||||
return roles
|
||||
|
||||
|
||||
def save_roles(roles):
|
||||
if ROLE_DB is None:
|
||||
return False
|
||||
raw = {
|
||||
node: role.__dict__
|
||||
for node, role in roles.items()
|
||||
}
|
||||
try:
|
||||
ROLE_DB.write_text(json.dumps(raw, indent=2))
|
||||
return True
|
||||
except (OSError, PermissionError):
|
||||
return False
|
||||
|
||||
|
||||
def get_role(node_name, roles):
|
||||
return roles.get(node_name, DEFAULT_ROLE)
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeState:
|
||||
"""
|
||||
Single source of truth for node health + capability.
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
# runtime
|
||||
cpu_load: float = 0.0
|
||||
memory_load: float = 0.0
|
||||
temp: float = 0.0
|
||||
|
||||
# scheduling intelligence
|
||||
score: float = 0.0
|
||||
instability: float = 0.0
|
||||
|
||||
# execution tracking
|
||||
active_jobs: int = 0
|
||||
last_seen: float = 0.0
|
||||
|
||||
# capabilities
|
||||
labels: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# health flags
|
||||
healthy: bool = True
|
||||
degraded: bool = False
|
||||
offline: bool = False
|
||||
|
||||
|
||||
class NodeStateRegistry:
|
||||
"""
|
||||
Central authority for node truth.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.nodes: Dict[str, NodeState] = {}
|
||||
|
||||
def update(self, name: str, **kwargs):
|
||||
if name not in self.nodes:
|
||||
self.nodes[name] = NodeState(name=name)
|
||||
|
||||
for k, v in kwargs.items():
|
||||
setattr(self.nodes[name], k, v)
|
||||
|
||||
def get(self, name: str) -> Optional[NodeState]:
|
||||
return self.nodes.get(name)
|
||||
|
||||
def all(self):
|
||||
return list(self.nodes.values())
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import json
|
||||
import requests
|
||||
from backend.config import CONFIG
|
||||
from backend.scheduler import build_distcc_hosts
|
||||
|
||||
AGENT_PORT = 8787
|
||||
|
||||
|
||||
def fetch_agent(node):
|
||||
try:
|
||||
r = requests.get(f"http://{node['host']}:{AGENT_PORT}/status", timeout=1.5)
|
||||
return r.json()
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def enrich(node):
|
||||
agent = fetch_agent(node)
|
||||
|
||||
return {
|
||||
"name": node["name"],
|
||||
"host": node["host"],
|
||||
"state": "online" if agent else "offline",
|
||||
"agent": agent,
|
||||
"release_ready": agent is not None
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
nodes = [enrich(n) for n in CONFIG["nodes"]]
|
||||
|
||||
output = {
|
||||
"master": CONFIG["master"],
|
||||
"nodes": nodes,
|
||||
"distcc": build_distcc_hosts(nodes)
|
||||
}
|
||||
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
"""
|
||||
PipelineEngine — executes a build DAG.
|
||||
|
||||
For each action:
|
||||
1. Schedule: pick best node via scheduler
|
||||
2. Emit task_update(scheduled) with deps + critical info
|
||||
3. Check cache: if hit, emit cache_update + skip execution
|
||||
4. Execute: emit task_update(running), run via runtime router, emit task_update(done|failed)
|
||||
5. On failure: emit failure event + stop downstream actions
|
||||
|
||||
Emits events on the bus so the UI / WS stream / cause graph / timeline store
|
||||
all pick them up.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from backend.events.bus import EventBus
|
||||
from backend.events.schema import EventType
|
||||
from backend.scheduler.optimizer import choose_best_node
|
||||
from backend.executor.runtime_router import execute_action
|
||||
from backend.cache.minio_cache import MinioCache
|
||||
from backend.graph.plan import build_action_graph
|
||||
from backend.graph.critical_path import compute_critical_path
|
||||
|
||||
|
||||
class PipelineEngine:
|
||||
"""Runs a build DAG against a cluster of nodes.
|
||||
|
||||
Constructor:
|
||||
nodes: list of node dicts (config + state)
|
||||
node_registry: NodeStateRegistry singleton (for live state lookups)
|
||||
event_bus: EventBus singleton (for emitting events)
|
||||
cache: optional MinioCache (or compatible). If None, no cache.
|
||||
build_id: optional build identifier (for correlating events)
|
||||
"""
|
||||
|
||||
def __init__(self, nodes, node_registry, event_bus: EventBus,
|
||||
cache: Optional[Any] = None, build_id: Optional[str] = None):
|
||||
self.nodes = nodes
|
||||
self.node_registry = node_registry
|
||||
self.bus = event_bus
|
||||
self.cache = cache # may be None
|
||||
self.build_id = build_id or f"build-{int(time.time())}"
|
||||
|
||||
# Pause / step state (used by debugger)
|
||||
self._paused = False
|
||||
self._step_mode = False
|
||||
self._step_event = asyncio.Event() if asyncio.get_event_loop() else None
|
||||
|
||||
# Results
|
||||
self.last_results: List[tuple] = []
|
||||
self.critical_path: Optional[Dict[str, Any]] = None
|
||||
|
||||
# -------------------------------------------------
|
||||
# DEBUGGER HOOKS
|
||||
# -------------------------------------------------
|
||||
def pause(self):
|
||||
self._paused = True
|
||||
|
||||
def resume(self):
|
||||
self._paused = False
|
||||
if self._step_event:
|
||||
self._step_event.set()
|
||||
|
||||
def step(self):
|
||||
"""Advance one action when paused."""
|
||||
self._step_mode = True
|
||||
if self._step_event:
|
||||
self._step_event.set()
|
||||
|
||||
# -------------------------------------------------
|
||||
# MAIN ENTRYPOINT
|
||||
# -------------------------------------------------
|
||||
async def run(self, project) -> List[tuple]:
|
||||
"""Build the project. Returns list of (action_name, state) tuples."""
|
||||
|
||||
actions = build_action_graph(project)
|
||||
self.critical_path = compute_critical_path(actions)
|
||||
critical_names = set(self.critical_path.get("score_map", {}).keys()) if self.critical_path else set()
|
||||
|
||||
self.last_results = []
|
||||
completed: Dict[str, str] = {} # name -> state
|
||||
failed_actions: set = set()
|
||||
|
||||
for action in actions:
|
||||
# Skip if any dependency failed
|
||||
deps = action.get("deps", [])
|
||||
failed_deps = [d for d in deps if completed.get(d) == "failed"]
|
||||
if failed_deps:
|
||||
self.bus.emit(
|
||||
EventType.TASK_UPDATE,
|
||||
action=action["name"],
|
||||
node=None,
|
||||
state="skipped",
|
||||
reason=f"deps_failed:{','.join(failed_deps)}",
|
||||
meta={
|
||||
"deps": deps,
|
||||
"critical": action["name"] in critical_names,
|
||||
"target": action.get("target", "native"),
|
||||
"build_id": self.build_id,
|
||||
},
|
||||
)
|
||||
completed[action["name"]] = "skipped"
|
||||
self.last_results.append((action["name"], "skipped"))
|
||||
continue
|
||||
|
||||
# Wait if paused
|
||||
await self._wait_if_paused()
|
||||
|
||||
# Schedule
|
||||
node = choose_best_node(self.nodes, action, self.node_registry)
|
||||
node_name = node["name"] if node else None
|
||||
|
||||
self.bus.emit(
|
||||
EventType.TASK_UPDATE,
|
||||
action=action["name"],
|
||||
node=node_name,
|
||||
state="scheduled",
|
||||
meta={
|
||||
"deps": deps,
|
||||
"critical": action["name"] in critical_names,
|
||||
"target": action.get("target", "native"),
|
||||
"build_id": self.build_id,
|
||||
},
|
||||
)
|
||||
|
||||
# Cache check
|
||||
cache_hit = False
|
||||
if self.cache and action.get("hash"):
|
||||
try:
|
||||
if self.cache.exists(action["hash"]):
|
||||
cache_hit = True
|
||||
self.bus.emit(
|
||||
EventType.CACHE_UPDATE,
|
||||
action=action["name"],
|
||||
node=node_name,
|
||||
state="hit",
|
||||
meta={"build_id": self.build_id},
|
||||
)
|
||||
except Exception:
|
||||
# Cache errors are non-fatal
|
||||
pass
|
||||
|
||||
# Emit running
|
||||
self.bus.emit(
|
||||
EventType.TASK_UPDATE,
|
||||
action=action["name"],
|
||||
node=node_name,
|
||||
state="running",
|
||||
meta={
|
||||
"deps": deps,
|
||||
"critical": action["name"] in critical_names,
|
||||
"target": action.get("target", "native"),
|
||||
"cache": "hit" if cache_hit else None,
|
||||
"build_id": self.build_id,
|
||||
},
|
||||
)
|
||||
|
||||
if not cache_hit:
|
||||
# Execute (in a thread so we don't block the event loop)
|
||||
# Pass action itself as workspace hint — execute_action reads
|
||||
# action["dir"] and action["env"] for proper cwd + env setup.
|
||||
try:
|
||||
rc = await asyncio.to_thread(
|
||||
execute_action, action, action.get("dir") or "/tmp",
|
||||
node or {"name": "localhost"}
|
||||
)
|
||||
state = "done" if rc == 0 else "failed"
|
||||
except Exception as e:
|
||||
state = "failed"
|
||||
rc = -1
|
||||
self.bus.emit(
|
||||
EventType.FAILURE,
|
||||
action=action["name"],
|
||||
node=node_name,
|
||||
state="failed",
|
||||
reason=f"execution_exception:{type(e).__name__}:{e}",
|
||||
meta={"build_id": self.build_id},
|
||||
)
|
||||
else:
|
||||
state = "done"
|
||||
rc = 0
|
||||
|
||||
# Emit terminal state
|
||||
self.bus.emit(
|
||||
EventType.TASK_UPDATE,
|
||||
action=action["name"],
|
||||
node=node_name,
|
||||
state=state,
|
||||
meta={
|
||||
"deps": deps,
|
||||
"critical": action["name"] in critical_names,
|
||||
"target": action.get("target", "native"),
|
||||
"cache": "hit" if cache_hit else None,
|
||||
"build_id": self.build_id,
|
||||
"rc": rc,
|
||||
},
|
||||
)
|
||||
|
||||
completed[action["name"]] = state
|
||||
self.last_results.append((action["name"], state))
|
||||
|
||||
if state == "failed":
|
||||
failed_actions.add(action["name"])
|
||||
|
||||
# Brief yield so the WS broadcast has time to fire
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
return self.last_results
|
||||
|
||||
# -------------------------------------------------
|
||||
# INTERNAL
|
||||
# -------------------------------------------------
|
||||
async def _wait_if_paused(self):
|
||||
if not self._paused:
|
||||
return
|
||||
# Wait until resumed (or stepped)
|
||||
if self._step_event is None:
|
||||
self._step_event = asyncio.Event()
|
||||
self._step_event.clear()
|
||||
while self._paused and not self._step_mode:
|
||||
await self._step_event.wait()
|
||||
self._step_event.clear()
|
||||
self._step_mode = False
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
"""
|
||||
Pipeline feedback — feeds execution results back into the policy engine
|
||||
so it can learn which nodes / actions succeed vs fail.
|
||||
|
||||
Currently the PolicyEngine doesn't have a `update_reputation` method
|
||||
(see backend/policy/engine.py for the simplified evaluate() interface).
|
||||
This module is kept as a stub for future learning-loop work.
|
||||
"""
|
||||
|
||||
from backend.policy.engine import PolicyEngine
|
||||
|
||||
# Singleton policy engine (db=None — no learning yet)
|
||||
policy = PolicyEngine(db=None)
|
||||
|
||||
|
||||
def report_execution(node, action, success, duration, temp_before, temp_after):
|
||||
"""Feed execution results back to the policy engine.
|
||||
|
||||
In the future this should update per-node reputation scores so the
|
||||
scheduler can prefer nodes that historically succeed and avoid nodes
|
||||
that overheat. For now it's a no-op logging shim.
|
||||
"""
|
||||
thermal_spike = (temp_after - temp_before) > 0.15
|
||||
|
||||
# Future: policy.update_reputation(node, success, duration, thermal_spike)
|
||||
# For now, just return the assessment
|
||||
return {
|
||||
"node": node,
|
||||
"action": action,
|
||||
"success": success,
|
||||
"duration": duration,
|
||||
"thermal_spike": thermal_spike,
|
||||
"temp_delta": temp_after - temp_before,
|
||||
}
|
||||
|
|
@ -0,0 +1,357 @@
|
|||
"""
|
||||
BuildRunner — manages build lifecycle.
|
||||
|
||||
Wraps PipelineEngine with:
|
||||
- Build ID generation
|
||||
- State tracking (queued, running, complete, failed)
|
||||
- Async task management (so /api/build returns immediately)
|
||||
- Optional persistence via a storage backend
|
||||
|
||||
Usage:
|
||||
runner = BuildRunner(bus, registry, nodes)
|
||||
build_id = await runner.start(BuildBody(cmd="make -j4", dir="/tmp/foo"))
|
||||
state = runner.state(build_id)
|
||||
all_builds = runner.list_builds()
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from backend.events.bus import EventBus
|
||||
from backend.events.schema import EventType
|
||||
from backend.pipeline.engine import PipelineEngine
|
||||
from backend.cache.minio_cache import MinioCache
|
||||
|
||||
|
||||
# Optional storage backend
|
||||
try:
|
||||
from backend.storage.sqlite_db import STORAGE
|
||||
except Exception:
|
||||
STORAGE = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildRecord:
|
||||
build_id: str
|
||||
cmd: str
|
||||
dir: str
|
||||
project: Optional[str] = None
|
||||
arch: str = "x86_64"
|
||||
target: str = "linux-gnu"
|
||||
toolchain: str = "gcc"
|
||||
started_at: float = field(default_factory=time.time)
|
||||
ended_at: Optional[float] = None
|
||||
status: str = "queued" # queued | running | complete | failed
|
||||
actions: List[Dict[str, str]] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class BuildRunner:
|
||||
"""Manages build lifecycle and tracks all builds in memory (or via storage)."""
|
||||
|
||||
def __init__(self, bus: EventBus, registry, nodes: List[Dict[str, Any]],
|
||||
storage=None):
|
||||
self.bus = bus
|
||||
self.registry = registry
|
||||
self.nodes = nodes
|
||||
self.storage = storage if storage is not None else STORAGE # may still be None
|
||||
self._builds: Dict[str, BuildRecord] = {}
|
||||
self._tasks: Dict[str, asyncio.Task] = {}
|
||||
self._engines: Dict[str, PipelineEngine] = {} # build_id -> engine
|
||||
self._cache: Optional[MinioCache] = None # lazy init
|
||||
|
||||
# On startup, load any persisted builds into the in-memory cache
|
||||
if self.storage:
|
||||
self.load_from_storage()
|
||||
|
||||
# Wire engines into debugger + pipeline_control so they can
|
||||
# pause/resume/step the most-recently-started build.
|
||||
self._wire_debugger()
|
||||
|
||||
def _wire_debugger(self):
|
||||
"""Register our engine getter with the debugger + pipeline_control modules."""
|
||||
try:
|
||||
from backend.api import debugger as dbg
|
||||
from backend.api import pipeline_control as pc
|
||||
# Replace their _ENGINE singletons with a proxy that always
|
||||
# points at the most recent running build's engine.
|
||||
proxy = _EngineProxy(self)
|
||||
dbg.set_engine(proxy)
|
||||
pc.set_engine(proxy)
|
||||
except Exception as e:
|
||||
# Don't crash if debugger/pipeline_control aren't importable
|
||||
pass
|
||||
|
||||
# -------------------------------------------------
|
||||
# CACHE
|
||||
# -------------------------------------------------
|
||||
def _get_cache(self) -> Optional[MinioCache]:
|
||||
"""Lazily construct a MinioCache. Returns None if MinIO is unreachable
|
||||
or not configured."""
|
||||
if self._cache is not None:
|
||||
return self._cache
|
||||
try:
|
||||
self._cache = MinioCache("localhost:9000", "minioadmin", "minioadmin")
|
||||
# Force connection test
|
||||
self._cache._ensure_client()
|
||||
return self._cache
|
||||
except Exception:
|
||||
self._cache = None
|
||||
return None
|
||||
|
||||
# -------------------------------------------------
|
||||
# START A BUILD
|
||||
# -------------------------------------------------
|
||||
async def start(self, body) -> str:
|
||||
"""Queue a build and return its ID. The actual run happens in a background task."""
|
||||
build_id = str(uuid.uuid4())[:8]
|
||||
record = BuildRecord(
|
||||
build_id=build_id,
|
||||
cmd=body.cmd,
|
||||
dir=body.dir,
|
||||
project=getattr(body, "project", None),
|
||||
arch=getattr(body, "arch", "x86_64"),
|
||||
target=getattr(body, "target", "linux-gnu"),
|
||||
toolchain=getattr(body, "toolchain", "gcc"),
|
||||
)
|
||||
record.status = "running"
|
||||
self._builds[build_id] = record
|
||||
self._persist(record)
|
||||
|
||||
# Launch in background
|
||||
task = asyncio.create_task(self._run_build(build_id))
|
||||
self._tasks[build_id] = task
|
||||
return build_id
|
||||
|
||||
# -------------------------------------------------
|
||||
# BUILD EXECUTION
|
||||
# -------------------------------------------------
|
||||
async def _run_build(self, build_id: str):
|
||||
record = self._builds[build_id]
|
||||
try:
|
||||
# Construct a project spec the engine can consume
|
||||
project = self._build_project_spec(record)
|
||||
|
||||
# Choose execution mode
|
||||
if record.cmd and record.cmd.startswith("make"):
|
||||
# Real build — use the actual engine
|
||||
engine = PipelineEngine(
|
||||
nodes=self.nodes,
|
||||
node_registry=self.registry,
|
||||
event_bus=self.bus,
|
||||
cache=self._get_cache(),
|
||||
build_id=build_id,
|
||||
)
|
||||
self._engines[build_id] = engine
|
||||
try:
|
||||
results = await engine.run(project)
|
||||
finally:
|
||||
# Engine done — drop from active set
|
||||
self._engines.pop(build_id, None)
|
||||
|
||||
# Record results
|
||||
record.actions = [{"name": n, "state": s} for n, s in results]
|
||||
record.status = "failed" if any(s == "failed" for _, s in results) else "complete"
|
||||
else:
|
||||
# Synthetic pipeline for non-shell build commands (demo mode)
|
||||
await self._run_synthetic(build_id, record)
|
||||
record.status = "complete"
|
||||
|
||||
except Exception as e:
|
||||
record.status = "failed"
|
||||
record.error = f"{type(e).__name__}: {e}"
|
||||
self.bus.emit(
|
||||
EventType.FAILURE,
|
||||
action=None,
|
||||
node=None,
|
||||
state="failed",
|
||||
reason=f"build_crashed:{record.error}",
|
||||
meta={"build_id": build_id},
|
||||
)
|
||||
finally:
|
||||
record.ended_at = time.time()
|
||||
self._persist(record)
|
||||
|
||||
def _build_project_spec(self, record: BuildRecord) -> Dict[str, Any]:
|
||||
"""Convert a BuildRecord into a project spec the engine understands."""
|
||||
return {
|
||||
"source": record.project or "local",
|
||||
"default_target": record.target,
|
||||
"build_dir": record.dir, # passed through to actions as `dir`
|
||||
"build_env": {
|
||||
"ARCH": record.arch,
|
||||
"TARGET": record.target,
|
||||
"CC": record.toolchain,
|
||||
"DISTCC_HOSTS": " ".join(
|
||||
n.get("host", "") for n in self.nodes if n.get("state") == "online"
|
||||
),
|
||||
},
|
||||
"targets": {
|
||||
record.target: record.cmd,
|
||||
},
|
||||
}
|
||||
|
||||
async def _run_synthetic(self, build_id: str, record: BuildRecord):
|
||||
"""Fallback synthetic pipeline — emits a realistic-looking stream of
|
||||
events without actually running any commands. Used when the build
|
||||
command isn't a real shell command (e.g. demo / smoke test)."""
|
||||
import random
|
||||
actions = [
|
||||
("fetch_source", []),
|
||||
("configure", ["fetch_source"]),
|
||||
("build_kernel", ["configure"]),
|
||||
("build_modules", ["configure"]),
|
||||
("build_initramfs", ["configure"]),
|
||||
("build_debian", ["build_kernel", "build_modules"]),
|
||||
("package", ["build_debian"]),
|
||||
("release", ["package"]),
|
||||
]
|
||||
critical = {"fetch_source", "configure", "build_kernel", "build_debian", "package", "release"}
|
||||
|
||||
for action, deps in actions:
|
||||
node_name = random.choice([n["name"] for n in self.nodes if n.get("state") != "offline"]) if self.nodes else "localhost"
|
||||
|
||||
self.bus.emit(EventType.TASK_UPDATE, action=action, node=node_name, state="scheduled",
|
||||
meta={"deps": deps, "critical": action in critical, "target": record.target, "build_id": build_id})
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
self.bus.emit(EventType.TASK_UPDATE, action=action, node=node_name, state="running",
|
||||
meta={"deps": deps, "critical": action in critical, "target": record.target, "build_id": build_id})
|
||||
await asyncio.sleep(random.uniform(0.4, 1.0))
|
||||
|
||||
cache_hit = random.random() < 0.25
|
||||
self.bus.emit(EventType.TASK_UPDATE, action=action, node=node_name, state="done",
|
||||
meta={"deps": deps, "cache": "hit" if cache_hit else None, "critical": action in critical, "target": record.target, "build_id": build_id})
|
||||
if cache_hit:
|
||||
self.bus.emit(EventType.CACHE_UPDATE, action=action, node=node_name, state="hit",
|
||||
meta={"build_id": build_id})
|
||||
record.actions.append({"name": action, "state": "done"})
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# -------------------------------------------------
|
||||
# QUERY
|
||||
# -------------------------------------------------
|
||||
def state(self, build_id: str) -> Optional[Dict[str, Any]]:
|
||||
rec = self._builds.get(build_id)
|
||||
if rec:
|
||||
return rec.to_dict()
|
||||
if self.storage:
|
||||
try:
|
||||
return self.storage.get_build(build_id)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def list_builds(self) -> List[Dict[str, Any]]:
|
||||
# Merge in-memory + storage (in-memory takes precedence)
|
||||
in_mem = {b.build_id: b.to_dict() for b in self._builds.values()}
|
||||
if not self.storage:
|
||||
return list(in_mem.values())
|
||||
try:
|
||||
stored = {b["build_id"]: b for b in self.storage.list_builds()}
|
||||
except Exception:
|
||||
stored = {}
|
||||
# Merge: in-memory wins (it has fresher state for running builds)
|
||||
merged = {**stored, **in_mem}
|
||||
return sorted(merged.values(), key=lambda b: b.get("started_at", 0), reverse=True)
|
||||
|
||||
def cancel(self, build_id: str) -> bool:
|
||||
task = self._tasks.get(build_id)
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
rec = self._builds.get(build_id)
|
||||
if rec:
|
||||
rec.status = "cancelled"
|
||||
rec.ended_at = time.time()
|
||||
self._persist(rec)
|
||||
return True
|
||||
return False
|
||||
|
||||
# -------------------------------------------------
|
||||
# DEBUGGER CONTROL
|
||||
# -------------------------------------------------
|
||||
def get_engine(self, build_id: Optional[str] = None) -> Optional[PipelineEngine]:
|
||||
"""Look up the running engine for a build.
|
||||
If build_id is None, returns the most recently started engine."""
|
||||
if build_id:
|
||||
return self._engines.get(build_id)
|
||||
# Return the most recent (last-inserted) engine
|
||||
if not self._engines:
|
||||
return None
|
||||
return list(self._engines.values())[-1]
|
||||
|
||||
def active_build_ids(self) -> List[str]:
|
||||
"""Return IDs of builds with engines currently running."""
|
||||
return list(self._engines.keys())
|
||||
|
||||
# -------------------------------------------------
|
||||
# PERSISTENCE
|
||||
# -------------------------------------------------
|
||||
def _persist(self, record: BuildRecord):
|
||||
if self.storage:
|
||||
try:
|
||||
self.storage.save_build(record.to_dict())
|
||||
except Exception:
|
||||
pass # storage errors are non-fatal
|
||||
|
||||
def load_from_storage(self):
|
||||
"""On startup, load any persisted builds into the in-memory cache."""
|
||||
if not self.storage:
|
||||
return
|
||||
try:
|
||||
builds = self.storage.list_builds()
|
||||
for b in builds:
|
||||
# Don't overwrite running builds (they're stale from a crashed process)
|
||||
if b.get("status") == "running":
|
||||
b["status"] = "crashed"
|
||||
b["error"] = "process restarted while build was running"
|
||||
rec = BuildRecord(**{k: v for k, v in b.items() if k in BuildRecord.__dataclass_fields__})
|
||||
self._builds[rec.build_id] = rec
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class _EngineProxy:
|
||||
"""Stand-in object that the debugger + pipeline_control modules can call
|
||||
pause/resume/step on. Forwards calls to the BuildRunner's most-recent
|
||||
active engine, so debugger buttons affect the currently-running build."""
|
||||
|
||||
def __init__(self, runner: BuildRunner):
|
||||
self._runner = runner
|
||||
|
||||
def _engine(self) -> Optional[PipelineEngine]:
|
||||
return self._runner.get_engine()
|
||||
|
||||
def pause(self):
|
||||
eng = self._engine()
|
||||
if eng:
|
||||
eng.pause()
|
||||
return True
|
||||
return False
|
||||
|
||||
def resume(self):
|
||||
eng = self._engine()
|
||||
if eng:
|
||||
eng.resume()
|
||||
return True
|
||||
return False
|
||||
|
||||
def step(self):
|
||||
eng = self._engine()
|
||||
if eng:
|
||||
eng.step()
|
||||
return True
|
||||
return False
|
||||
|
||||
def step_back(self):
|
||||
# Step-back is replay-only; live engine can't undo
|
||||
return False
|
||||
|
||||
def is_active(self) -> bool:
|
||||
return self._engine() is not None
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
from enum import Enum
|
||||
|
||||
|
||||
class State(Enum):
|
||||
PENDING = "pending"
|
||||
PREFLIGHT = "preflight"
|
||||
SCHEDULED = "scheduled"
|
||||
EXECUTING = "executing"
|
||||
|
||||
CACHE_HIT = "cache_hit"
|
||||
DONE = "done"
|
||||
|
||||
FAILED = "failed"
|
||||
RETRY = "retry"
|
||||
DEAD = "dead"
|
||||
|
||||
THERMAL_GATED = "thermal_gated"
|
||||
VM_ESCALATED = "vm_escalated"
|
||||
POLICY_HELD = "policy_held"
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
from backend.events.emitter import emit_policy
|
||||
|
||||
POLICY = None
|
||||
|
||||
|
||||
def init(policy_engine):
|
||||
global POLICY
|
||||
POLICY = policy_engine
|
||||
|
||||
|
||||
async def set_override(data):
|
||||
|
||||
POLICY.set_override(data["key"], data["value"])
|
||||
|
||||
await emit_policy(data["key"], data["value"])
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
async def clear_override(data):
|
||||
|
||||
POLICY.clear_override(data["key"])
|
||||
|
||||
await emit_policy(data["key"], None)
|
||||
|
||||
return {"status": "cleared"}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
"""
|
||||
PolicyEngine — applies static rules + learned heuristics + live overrides
|
||||
to scheduler decisions.
|
||||
|
||||
Constructor accepts an optional `db` (anything with an `events` dict-like
|
||||
attribute). If db is None, the heuristics layer is skipped.
|
||||
"""
|
||||
|
||||
|
||||
class PolicyEngine:
|
||||
|
||||
def __init__(self, db=None):
|
||||
self.db = db
|
||||
self.rules = []
|
||||
self.overrides = {}
|
||||
|
||||
# -----------------------------
|
||||
# REGISTER RULE
|
||||
# -----------------------------
|
||||
def add_rule(self, rule):
|
||||
self.rules.append(rule)
|
||||
|
||||
# -----------------------------
|
||||
# SET LIVE OVERRIDE
|
||||
# -----------------------------
|
||||
def set_override(self, key, value):
|
||||
self.overrides[key] = value
|
||||
|
||||
# -----------------------------
|
||||
# CLEAR OVERRIDE
|
||||
# -----------------------------
|
||||
def clear_override(self, key):
|
||||
if key in self.overrides:
|
||||
del self.overrides[key]
|
||||
|
||||
# -----------------------------
|
||||
# APPLY POLICY TO NODE SELECTION
|
||||
# -----------------------------
|
||||
def evaluate(self, action, target, node):
|
||||
score_modifier = 0
|
||||
for rule in self.rules:
|
||||
try:
|
||||
score_modifier += rule.apply(action, target, node)
|
||||
except Exception:
|
||||
pass
|
||||
score_modifier += self._heuristics(action, target, node)
|
||||
score_modifier += self._overrides(action, target, node)
|
||||
return score_modifier
|
||||
|
||||
# -----------------------------
|
||||
# HEURISTICS (LEARNING LAYER)
|
||||
# -----------------------------
|
||||
def _heuristics(self, action, target, node):
|
||||
if self.db is None:
|
||||
return 0
|
||||
if not hasattr(self.db, "events"):
|
||||
return 0
|
||||
try:
|
||||
history = self.db.events
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
score = 0
|
||||
success_count = 0
|
||||
# history may be dict-of-lists or a flat list
|
||||
if isinstance(history, dict):
|
||||
iterable = (e for sess in history.values() for e in (sess or []))
|
||||
else:
|
||||
iterable = history or []
|
||||
|
||||
for event in iterable:
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
if event.get("type") in ("action_end", "task_update"):
|
||||
d = event.get("data", event)
|
||||
if (d.get("action") == action.get("name")
|
||||
and d.get("state") == "done"
|
||||
and d.get("node") == node.get("name")):
|
||||
success_count += 1
|
||||
score += success_count * 2
|
||||
return score
|
||||
|
||||
# -----------------------------
|
||||
# LIVE OVERRIDES
|
||||
# -----------------------------
|
||||
def _overrides(self, action, target, node):
|
||||
score = 0
|
||||
forced = self.overrides.get("force_node")
|
||||
if forced and node.get("name") == forced:
|
||||
score += 1000
|
||||
avoid = self.overrides.get("avoid_node")
|
||||
if avoid and node.get("name") == avoid:
|
||||
score -= 1000
|
||||
return score
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
class Rule:
|
||||
|
||||
def apply(self, action, target, node):
|
||||
return 0
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# EXAMPLES
|
||||
# -----------------------------
|
||||
|
||||
class AvoidHotNodes(Rule):
|
||||
|
||||
def apply(self, action, target, node):
|
||||
|
||||
if node.get("temp", 0) > 80:
|
||||
return -100
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class PreferLXCForARM(Rule):
|
||||
|
||||
def apply(self, action, target, node):
|
||||
|
||||
if target.arch == "arm64" and target.runtime == "lxc":
|
||||
return 20
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class AvoidVMForSmallBuilds(Rule):
|
||||
|
||||
def apply(self, action, target, node):
|
||||
|
||||
if target.runtime == "libvirt" and action.get("size") == "small":
|
||||
return -50
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class SpreadLoad(Rule):
|
||||
|
||||
def apply(self, action, target, node):
|
||||
|
||||
load = node.get("cpu_load", 50)
|
||||
|
||||
return -(load // 2)
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
from backend.governor import thermal_cap
|
||||
from backend.cache_local import make_build_key, cache_hit
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# analyze cluster health
|
||||
# ----------------------------
|
||||
def cluster_heat(nodes):
|
||||
if not nodes:
|
||||
return 0.5
|
||||
|
||||
total = 0
|
||||
count = 0
|
||||
|
||||
for n in nodes:
|
||||
agent = n.get("agent") or {}
|
||||
load = agent.get("load", "1 1 1")
|
||||
|
||||
try:
|
||||
total += float(load.split()[0])
|
||||
except:
|
||||
total += 1.0
|
||||
|
||||
count += 1
|
||||
|
||||
return total / max(count, 1)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# thermal risk scoring
|
||||
# ----------------------------
|
||||
def risk_level(node):
|
||||
agent = node.get("agent") or {}
|
||||
|
||||
cap = thermal_cap(agent)
|
||||
|
||||
if cap < 0.3:
|
||||
return "HIGH_RISK"
|
||||
elif cap < 0.6:
|
||||
return "MODERATE"
|
||||
return "SAFE"
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# predict build cost
|
||||
# ----------------------------
|
||||
def estimate_build_cost(nodes):
|
||||
cost = 0.0
|
||||
|
||||
for n in nodes:
|
||||
agent = n.get("agent") or {}
|
||||
load = agent.get("load", "1 1 1")
|
||||
|
||||
try:
|
||||
cost += float(load.split()[0])
|
||||
except:
|
||||
cost += 1.0
|
||||
|
||||
return cost / max(len(nodes), 1)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# recommendation engine
|
||||
# ----------------------------
|
||||
def recommend(project, snapshot, nodes):
|
||||
recommendations = {
|
||||
"build_now": True,
|
||||
"risk": "SAFE",
|
||||
"cache_likely": False,
|
||||
"preferred_nodes": [],
|
||||
"reason": []
|
||||
}
|
||||
|
||||
# ----------------------------
|
||||
# cache prediction
|
||||
# ----------------------------
|
||||
key_sample = make_build_key(
|
||||
snapshot["hash"],
|
||||
list(project["targets"].keys())[0],
|
||||
project.get("build_env", {})
|
||||
)
|
||||
|
||||
if cache_hit(key_sample):
|
||||
recommendations["cache_likely"] = True
|
||||
recommendations["reason"].append("Cache hit likely for current snapshot")
|
||||
|
||||
# ----------------------------
|
||||
# cluster load analysis
|
||||
# ----------------------------
|
||||
heat = cluster_heat(nodes)
|
||||
|
||||
if heat > 3.0:
|
||||
recommendations["build_now"] = False
|
||||
recommendations["reason"].append("Cluster overloaded - recommend delay")
|
||||
|
||||
# ----------------------------
|
||||
# node filtering
|
||||
# ----------------------------
|
||||
safe_nodes = []
|
||||
|
||||
for n in nodes:
|
||||
if risk_level(n) == "SAFE":
|
||||
safe_nodes.append(n["name"])
|
||||
|
||||
recommendations["preferred_nodes"] = safe_nodes
|
||||
|
||||
# ----------------------------
|
||||
# final risk classification
|
||||
# ----------------------------
|
||||
if heat > 4.0:
|
||||
recommendations["risk"] = "HIGH_RISK"
|
||||
elif heat > 2.5:
|
||||
recommendations["risk"] = "MODERATE"
|
||||
|
||||
return recommendations
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
import json
|
||||
from backend.snapshot import create_snapshot
|
||||
from backend.recommend import recommend
|
||||
|
||||
|
||||
def run(project, nodes):
|
||||
snapshot = create_snapshot(project["source"])
|
||||
|
||||
result = recommend(project, snapshot, nodes)
|
||||
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
project = json.load(open(sys.argv[1]))
|
||||
nodes = json.load(open(sys.argv[2]))
|
||||
|
||||
run(project, nodes)
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
from backend.governor import thermal_cap
|
||||
|
||||
|
||||
def choose_nodes(nodes, target):
|
||||
"""
|
||||
Unified default mode:
|
||||
all nodes participate unless thermally restricted
|
||||
"""
|
||||
|
||||
usable = []
|
||||
|
||||
for n in nodes:
|
||||
agent = n.get("agent") or {}
|
||||
|
||||
if thermal_cap(agent) < 0.3:
|
||||
continue
|
||||
|
||||
usable.append(n)
|
||||
|
||||
return usable
|
||||
|
||||
|
||||
def plan_release(project, nodes):
|
||||
plan = {}
|
||||
|
||||
for target in project["targets"]:
|
||||
plan[target] = choose_nodes(nodes, target)
|
||||
|
||||
return plan
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
import os
|
||||
import json
|
||||
from backend.snapshot import create_snapshot
|
||||
from backend.scheduler import build_distcc_hosts
|
||||
from backend.config import CONFIG
|
||||
|
||||
|
||||
def run(cmd, cwd, env):
|
||||
import subprocess
|
||||
|
||||
p = subprocess.Popen(
|
||||
cmd,
|
||||
shell=True,
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True
|
||||
)
|
||||
|
||||
for line in p.stdout:
|
||||
print(line, end="")
|
||||
|
||||
p.wait()
|
||||
return p.returncode
|
||||
|
||||
|
||||
def build_target(project, snapshot, target, nodes):
|
||||
env = os.environ.copy()
|
||||
env["DISTCC_HOSTS"] = build_distcc_hosts(nodes)
|
||||
|
||||
cmd = project["targets"][target]
|
||||
|
||||
print(f"[FESTER] target={target}")
|
||||
print(f"[FESTER] snapshot={snapshot['hash']}")
|
||||
|
||||
return run(cmd, snapshot["path"], env)
|
||||
|
||||
|
||||
def run_release(project):
|
||||
snapshot = create_snapshot(project["source"])
|
||||
nodes = CONFIG["nodes"]
|
||||
|
||||
results = {}
|
||||
|
||||
for target in project["targets"]:
|
||||
results[target] = build_target(project, snapshot, target, nodes)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], "r") as f:
|
||||
project = json.load(f)
|
||||
|
||||
run_release(project)
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
"""
|
||||
backend.scheduler package — re-exports for backward compatibility.
|
||||
|
||||
The original `backend/scheduler.py` module has been moved to
|
||||
`scheduler_legacy.py` to resolve the package/module name collision.
|
||||
|
||||
This file re-exports the legacy symbols so existing callers
|
||||
(`from backend.scheduler import build_distcc_hosts`, `score_node`)
|
||||
keep working without changes.
|
||||
"""
|
||||
|
||||
from backend.scheduler_legacy import score_node, build_distcc_hosts
|
||||
from backend.scheduler.optimizer import choose_best_node
|
||||
|
||||
__all__ = ["score_node", "build_distcc_hosts", "choose_best_node"]
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
from backend.cache_local import cache_hit
|
||||
from backend.nodes.roles_store import get_role
|
||||
|
||||
|
||||
def score_node(node, role, cache_key):
|
||||
"""
|
||||
Higher score = better node
|
||||
"""
|
||||
|
||||
agent = node.get("agent", {})
|
||||
load = float(agent.get("load", "1 1 1").split()[0])
|
||||
|
||||
score = 100.0
|
||||
|
||||
# ----------------------------
|
||||
# thermal penalty
|
||||
# ----------------------------
|
||||
score -= load * 10
|
||||
|
||||
# ----------------------------
|
||||
# role weighting
|
||||
# ----------------------------
|
||||
score *= role.compile_weight
|
||||
|
||||
# ----------------------------
|
||||
# cache preference boost
|
||||
# ----------------------------
|
||||
if role.cache_reader and cache_hit(cache_key):
|
||||
score += 50
|
||||
|
||||
# ----------------------------
|
||||
# penalize overheating nodes
|
||||
# ----------------------------
|
||||
if load > role.max_thermal_state * 4:
|
||||
score -= 100
|
||||
|
||||
return score
|
||||
|
||||
|
||||
def select_nodes(nodes, roles, cache_key):
|
||||
ranked = []
|
||||
|
||||
for n in nodes:
|
||||
role = get_role(n["name"], roles)
|
||||
|
||||
score = score_node(n, role, cache_key)
|
||||
|
||||
ranked.append((score, n["name"]))
|
||||
|
||||
ranked.sort(reverse=True)
|
||||
|
||||
return [name for score, name in ranked]
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import requests
|
||||
|
||||
|
||||
PROM_URL = "http://localhost:9090"
|
||||
|
||||
|
||||
def get_node_load(node):
|
||||
query = f'node_load1{{instance="{node}"}}'
|
||||
|
||||
try:
|
||||
r = requests.get(
|
||||
f"{PROM_URL}/api/v1/query",
|
||||
params={"query": query}
|
||||
)
|
||||
|
||||
data = r.json()
|
||||
|
||||
return float(
|
||||
data["data"]["result"][0]["value"][1]
|
||||
)
|
||||
except:
|
||||
return 1.0
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
from backend.nodes.state_model import NodeStateRegistry
|
||||
|
||||
|
||||
def choose_best_node(nodes, action, registry: NodeStateRegistry = None):
|
||||
|
||||
best_node = None
|
||||
best_score = -999999
|
||||
|
||||
for node in nodes:
|
||||
|
||||
state = registry.get(node["name"]) if registry else None
|
||||
|
||||
cpu_load = state.cpu_load if state else node.get("cpu_load", 0)
|
||||
temp = state.temp if state else node.get("temp", 0)
|
||||
|
||||
score = 0
|
||||
score += (100 - cpu_load)
|
||||
score -= temp
|
||||
|
||||
if node.get("policy") == "preferred":
|
||||
score += 20
|
||||
|
||||
if node.get("policy") == "avoid":
|
||||
score -= 50
|
||||
|
||||
if state:
|
||||
score -= state.instability * 10
|
||||
|
||||
node["_computed_score"] = score
|
||||
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_node = node
|
||||
|
||||
return best_node
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
from backend.policy.engine import PolicyEngine
|
||||
|
||||
|
||||
class TargetOptimizer:
|
||||
|
||||
def __init__(self, nodes, policy_engine):
|
||||
self.nodes = nodes
|
||||
self.policy = policy_engine
|
||||
|
||||
def choose(self, action, target):
|
||||
|
||||
candidates = []
|
||||
|
||||
for node in self.nodes:
|
||||
|
||||
if not self._compatible(node, target):
|
||||
continue
|
||||
|
||||
base_score = self._score(node)
|
||||
|
||||
policy_score = self.policy.evaluate(action, target, node)
|
||||
|
||||
total = base_score + policy_score
|
||||
|
||||
candidates.append((node, total))
|
||||
|
||||
if not candidates:
|
||||
raise Exception("No compatible nodes")
|
||||
|
||||
candidates.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
return candidates[0][0]
|
||||
|
||||
def _compatible(self, node, target):
|
||||
|
||||
if node["arch"] != target.arch:
|
||||
if target.runtime == "host":
|
||||
return False
|
||||
|
||||
if target.runtime not in node["runtimes"]:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _score(self, node):
|
||||
|
||||
score = 0
|
||||
|
||||
score += (100 - node.get("cpu_load", 50))
|
||||
score += (100 - node.get("temp", 60))
|
||||
|
||||
return score
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from backend.governor import thermal_cap
|
||||
|
||||
|
||||
def score_node(node):
|
||||
if node["state"] == "offline":
|
||||
return 0.0
|
||||
|
||||
agent = node.get("agent") or {}
|
||||
|
||||
try:
|
||||
load = float(agent.get("load", "1 1 1").split()[0])
|
||||
except:
|
||||
load = 1.0
|
||||
|
||||
base = 10.0
|
||||
capacity = node.get("max_jobs", 8)
|
||||
|
||||
score = (base + capacity) - (load * 3.0)
|
||||
|
||||
return max(score, 0.1) * thermal_cap(agent)
|
||||
|
||||
|
||||
def build_distcc_hosts(nodes):
|
||||
scored = []
|
||||
|
||||
for n in nodes:
|
||||
scored.append((n, score_node(n)))
|
||||
|
||||
total = sum(s for _, s in scored) or 1.0
|
||||
|
||||
hosts = []
|
||||
|
||||
for node, score in scored:
|
||||
if node["state"] == "offline":
|
||||
continue
|
||||
|
||||
weight = int((score / total) * 64)
|
||||
weight = max(1, weight)
|
||||
|
||||
hosts.append(f"{node['host']}/{weight}")
|
||||
|
||||
return " ".join(hosts)
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
"""
|
||||
SessionDB — replay session storage.
|
||||
|
||||
Originally in-memory only; now backed by SQLite (via backend.storage.sqlite_db).
|
||||
Maintains the same API for backward compatibility with session.session.BuildSession.
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List
|
||||
|
||||
# Optional storage backend
|
||||
try:
|
||||
from backend.storage.sqlite_db import STORAGE
|
||||
except Exception:
|
||||
STORAGE = None
|
||||
|
||||
|
||||
class SessionDB:
|
||||
"""Replay session registry + event log.
|
||||
|
||||
All writes go to SQLite (if available) AND to an in-memory cache.
|
||||
Reads hit the in-memory cache first; if missing, fall back to SQLite.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# In-memory caches (fast path)
|
||||
self.sessions: Dict[str, Dict[str, Any]] = {}
|
||||
self.events: Dict[str, List[Dict[str, Any]]] = {}
|
||||
self.snapshots: Dict[str, List[Dict[str, Any]]] = {}
|
||||
|
||||
# On startup, load any persisted sessions
|
||||
if STORAGE:
|
||||
self._load_from_storage()
|
||||
|
||||
def _load_from_storage(self):
|
||||
try:
|
||||
for s in STORAGE.list_sessions():
|
||||
sid = s["session_id"]
|
||||
self.sessions[sid] = {
|
||||
"project": s.get("source", "live"),
|
||||
"created_at": s.get("created_at", time.time()),
|
||||
"status": "active",
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -----------------------------
|
||||
# CREATE SESSION
|
||||
# -----------------------------
|
||||
def create_session(self, project) -> str:
|
||||
sid = str(uuid.uuid4())
|
||||
|
||||
self.sessions[sid] = {
|
||||
"project": project,
|
||||
"created_at": time.time(),
|
||||
"status": "active",
|
||||
}
|
||||
self.events[sid] = []
|
||||
self.snapshots[sid] = []
|
||||
|
||||
if STORAGE:
|
||||
try:
|
||||
STORAGE.create_session(sid, source=str(project) if project else "live")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return sid
|
||||
|
||||
# -----------------------------
|
||||
# EVENT LOGGING (IMMUTABLE)
|
||||
# -----------------------------
|
||||
def log_event(self, session_id, event_type, payload):
|
||||
event = {
|
||||
"ts": time.time(),
|
||||
"type": event_type,
|
||||
"data": payload,
|
||||
}
|
||||
|
||||
self.events.setdefault(session_id, []).append(event)
|
||||
|
||||
if STORAGE:
|
||||
try:
|
||||
STORAGE.append_event(event, session_id=session_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return event
|
||||
|
||||
# -----------------------------
|
||||
# SNAPSHOT (STATE CHECKPOINT)
|
||||
# -----------------------------
|
||||
def snapshot(self, session_id, state):
|
||||
snap = {
|
||||
"ts": time.time(),
|
||||
"state": state,
|
||||
}
|
||||
|
||||
self.snapshots.setdefault(session_id, []).append(snap)
|
||||
# Note: snapshots are not persisted to SQLite (they're rare and large)
|
||||
return snap
|
||||
|
||||
# -----------------------------
|
||||
# GET FULL TIMELINE
|
||||
# -----------------------------
|
||||
def get_session(self, session_id):
|
||||
# Try in-memory first
|
||||
if session_id in self.sessions:
|
||||
return {
|
||||
"session": self.sessions.get(session_id),
|
||||
"events": self.events.get(session_id, []),
|
||||
"snapshots": self.snapshots.get(session_id, []),
|
||||
}
|
||||
|
||||
# Fall back to storage
|
||||
if STORAGE:
|
||||
try:
|
||||
sess = STORAGE.get_session(session_id)
|
||||
if sess:
|
||||
events = STORAGE.get_events(session_id)
|
||||
return {
|
||||
"session": {
|
||||
"project": sess.get("source"),
|
||||
"created_at": sess.get("created_at"),
|
||||
"status": "active",
|
||||
},
|
||||
"events": events,
|
||||
"snapshots": [],
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"session": None,
|
||||
"events": [],
|
||||
"snapshots": [],
|
||||
}
|
||||
|
||||
# -----------------------------
|
||||
# REPLAY ENGINE INPUT
|
||||
# -----------------------------
|
||||
def get_replay_stream(self, session_id):
|
||||
# In-memory first
|
||||
if session_id in self.events:
|
||||
return sorted(self.events[session_id], key=lambda e: e["ts"])
|
||||
# Storage fallback
|
||||
if STORAGE:
|
||||
try:
|
||||
return STORAGE.get_events(session_id)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
# -----------------------------
|
||||
# LIST ALL SESSIONS
|
||||
# -----------------------------
|
||||
def list_sessions(self) -> List[Dict[str, Any]]:
|
||||
seen = set()
|
||||
out = []
|
||||
# In-memory
|
||||
for sid, s in self.sessions.items():
|
||||
out.append({
|
||||
"id": sid,
|
||||
"source": s.get("project", "live"),
|
||||
"created_at": s.get("created_at"),
|
||||
"events": len(self.events.get(sid, [])),
|
||||
})
|
||||
seen.add(sid)
|
||||
# Storage
|
||||
if STORAGE:
|
||||
try:
|
||||
for s in STORAGE.list_sessions():
|
||||
if s["session_id"] not in seen:
|
||||
out.append({
|
||||
"id": s["session_id"],
|
||||
"source": s.get("source", "live"),
|
||||
"created_at": s.get("created_at"),
|
||||
"events": s.get("event_count", 0),
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
# Sort by created_at desc
|
||||
out.sort(key=lambda s: s.get("created_at", 0), reverse=True)
|
||||
return out
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from backend.session.db import SessionDB
|
||||
|
||||
|
||||
class SessionReplay:
|
||||
|
||||
def __init__(self, db: SessionDB):
|
||||
self.db = db
|
||||
|
||||
# -----------------------------
|
||||
# REPLAY SESSION
|
||||
# -----------------------------
|
||||
def replay(self, session_id):
|
||||
|
||||
stream = self.db.get_replay_stream(session_id)
|
||||
|
||||
for event in stream:
|
||||
|
||||
yield {
|
||||
"ts": event["ts"],
|
||||
"type": event["type"],
|
||||
"data": event["data"]
|
||||
}
|
||||
|
||||
# -----------------------------
|
||||
# REPLAY FROM SNAPSHOT
|
||||
# -----------------------------
|
||||
def replay_from_snapshot(self, session_id, index=0):
|
||||
|
||||
snapshots = self.db.snapshots[session_id]
|
||||
|
||||
if not snapshots:
|
||||
return []
|
||||
|
||||
base = snapshots[index]
|
||||
|
||||
return {
|
||||
"snapshot": base,
|
||||
"events_after": [
|
||||
e for e in self.db.events[session_id]
|
||||
if e["ts"] >= base["ts"]
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
from backend.session.db import SessionDB
|
||||
from backend.integrations.tmux import TmuxManager
|
||||
|
||||
|
||||
class BuildSession:
|
||||
|
||||
def __init__(self, project, nodes, db: SessionDB):
|
||||
|
||||
self.project = project
|
||||
self.nodes = nodes
|
||||
|
||||
self.db = db
|
||||
self.session_id = db.create_session(project)
|
||||
|
||||
self.tmux = TmuxManager()
|
||||
|
||||
# -----------------------------
|
||||
# REGISTER ACTION RUNTIME
|
||||
# -----------------------------
|
||||
def attach_action_runtime(self, action, node):
|
||||
|
||||
cmd = action.get("command", "make -j")
|
||||
|
||||
session_name = self.tmux.create_session(action, cmd)
|
||||
|
||||
self.db.log_event(
|
||||
self.session_id,
|
||||
"tmux_spawn",
|
||||
{
|
||||
"action": action["name"],
|
||||
"node": node["name"],
|
||||
"session": session_name
|
||||
}
|
||||
)
|
||||
|
||||
return session_name
|
||||
|
||||
# -----------------------------
|
||||
# SNAPSHOT HOOK
|
||||
# -----------------------------
|
||||
def snapshot(self, state):
|
||||
|
||||
return self.db.snapshot(
|
||||
self.session_id,
|
||||
state
|
||||
)
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# utility: run command
|
||||
# ----------------------------
|
||||
def run(cmd, cwd="/"):
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
shell=True,
|
||||
cwd=cwd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True
|
||||
)
|
||||
|
||||
out = []
|
||||
for line in process.stdout:
|
||||
out.append(line)
|
||||
|
||||
process.wait()
|
||||
return "".join(out), process.returncode
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# git (Forgejo primary)
|
||||
# ----------------------------
|
||||
def snapshot_git(repo_url):
|
||||
workdir = tempfile.mkdtemp(prefix="fester-snap-git-")
|
||||
|
||||
run(f"git clone --depth 1 {repo_url} {workdir}")
|
||||
|
||||
return workdir
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# hg adapter (snapshot only)
|
||||
# ----------------------------
|
||||
def snapshot_hg(repo_url):
|
||||
workdir = tempfile.mkdtemp(prefix="fester-snap-hg-")
|
||||
|
||||
run(f"hg clone {repo_url} {workdir}")
|
||||
|
||||
return workdir
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# svn adapter (export only)
|
||||
# ----------------------------
|
||||
def snapshot_svn(repo_url):
|
||||
workdir = tempfile.mkdtemp(prefix="fester-snap-svn-")
|
||||
|
||||
run(f"svn checkout {repo_url} {workdir}")
|
||||
|
||||
return workdir
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# cvs adapter (legacy dump)
|
||||
# ----------------------------
|
||||
def snapshot_cvs(repo_url):
|
||||
workdir = tempfile.mkdtemp(prefix="fester-snap-cvs-")
|
||||
|
||||
# best-effort export only
|
||||
run(f"cvs export -d {workdir} {repo_url}")
|
||||
|
||||
return workdir
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# unified entry point
|
||||
# ----------------------------
|
||||
def create_snapshot(source):
|
||||
kind = source["type"]
|
||||
url = source["url"]
|
||||
|
||||
if kind == "git":
|
||||
path = snapshot_git(url)
|
||||
elif kind == "hg":
|
||||
path = snapshot_hg(url)
|
||||
elif kind == "svn":
|
||||
path = snapshot_svn(url)
|
||||
elif kind == "cvs":
|
||||
path = snapshot_cvs(url)
|
||||
else:
|
||||
raise Exception(f"Unsupported VCS type: {kind}")
|
||||
|
||||
return fingerprint(path)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# deterministic fingerprint
|
||||
# ----------------------------
|
||||
def fingerprint(path):
|
||||
sha = hashlib.sha256()
|
||||
|
||||
for root, dirs, files in os.walk(path):
|
||||
for f in sorted(files):
|
||||
fp = os.path.join(root, f)
|
||||
try:
|
||||
with open(fp, "rb") as fh:
|
||||
sha.update(fh.read())
|
||||
except:
|
||||
pass
|
||||
|
||||
return {
|
||||
"path": path,
|
||||
"hash": sha.hexdigest(),
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z"
|
||||
}
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
"""
|
||||
Btrfs content-addressable storage with CoW reflink.
|
||||
|
||||
Uses `cp --reflink=always` to create copy-on-write reflinks on btrfs
|
||||
filesystems (instant copy, zero extra disk usage until divergence).
|
||||
|
||||
Falls back to:
|
||||
1. `cp --reflink=auto` (let filesystem decide — works on btrfs, falls
|
||||
back to normal copy on ext4/xfs without error)
|
||||
2. `shutil.copy2` (preserves metadata)
|
||||
3. `shutil.copy` (last resort)
|
||||
|
||||
Also exposes `is_btrfs(path)` to detect whether a path is on btrfs.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def is_btrfs(path) -> bool:
|
||||
"""Return True if `path` lives on a btrfs filesystem."""
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
["stat", "-f", "-c", "%T", str(path)],
|
||||
stderr=subprocess.DEVNULL,
|
||||
).decode().strip()
|
||||
return out.lower() == "btrfs"
|
||||
except (subprocess.CalledProcessError, FileNotFoundError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def store_reflink(src, dst) -> bool:
|
||||
"""
|
||||
Copy `src` to `dst` using CoW reflink when possible.
|
||||
|
||||
Returns True if a true reflink was created, False if a regular copy
|
||||
was used as fallback.
|
||||
"""
|
||||
src = str(src)
|
||||
dst = str(dst)
|
||||
|
||||
# Ensure destination directory exists
|
||||
Path(dst).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Try true reflink first (btrfs only)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["cp", "--reflink=always", src, dst],
|
||||
capture_output=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return True
|
||||
# If --reflink=always fails, the source/dest aren't on the same btrfs
|
||||
except FileNotFoundError:
|
||||
pass # cp not available (unlikely on Linux)
|
||||
|
||||
# Fall back to auto-reflink (works on btrfs without error, regular copy elsewhere)
|
||||
try:
|
||||
subprocess.run(
|
||||
["cp", "--reflink=auto", src, dst],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
# Preserve metadata
|
||||
shutil.copystat(src, dst)
|
||||
return False
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
pass
|
||||
|
||||
# Last resort: pure Python copy
|
||||
try:
|
||||
shutil.copy2(src, dst)
|
||||
except Exception:
|
||||
shutil.copy(src, dst)
|
||||
return False
|
||||
|
||||
|
||||
def reflink_snapshot(src_dir, dst_dir) -> dict:
|
||||
"""
|
||||
Snapshot an entire directory tree using CoW reflinks.
|
||||
|
||||
Returns a dict with:
|
||||
- reflinked: count of files that got true reflinks
|
||||
- copied: count of files that fell back to regular copy
|
||||
- failed: count of files that couldn't be copied
|
||||
- total_bytes: total size of files snapshotted
|
||||
"""
|
||||
src = Path(src_dir)
|
||||
dst = Path(dst_dir)
|
||||
if not src.is_dir():
|
||||
return {"reflinked": 0, "copied": 0, "failed": 0, "total_bytes": 0, "error": "src not a dir"}
|
||||
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
reflinked = copied = failed = 0
|
||||
total_bytes = 0
|
||||
|
||||
for root, dirs, files in os.walk(src):
|
||||
rel = Path(root).relative_to(src)
|
||||
(dst / rel).mkdir(parents=True, exist_ok=True)
|
||||
for f in files:
|
||||
src_file = Path(root) / f
|
||||
dst_file = dst / rel / f
|
||||
try:
|
||||
size = src_file.stat().st_size
|
||||
if store_reflink(src_file, dst_file):
|
||||
reflinked += 1
|
||||
else:
|
||||
copied += 1
|
||||
total_bytes += size
|
||||
except Exception:
|
||||
failed += 1
|
||||
|
||||
return {
|
||||
"reflinked": reflinked,
|
||||
"copied": copied,
|
||||
"failed": failed,
|
||||
"total_bytes": total_bytes,
|
||||
}
|
||||
|
||||
|
||||
def cas_path(config, key) -> str:
|
||||
"""Resolve the CAS storage path for a given key."""
|
||||
base = Path(config.get("btrfs_path", "/var/lib/fester/cas"))
|
||||
path = base / key
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
return str(path)
|
||||
|
||||
|
||||
def cas_store(config, key, src_path) -> dict:
|
||||
"""Store a file/directory in the CAS under `key` using reflink.
|
||||
Returns the snapshot stats."""
|
||||
dst = cas_path(config, key)
|
||||
src = Path(src_path)
|
||||
if src.is_dir():
|
||||
return reflink_snapshot(src, dst)
|
||||
else:
|
||||
reflinked = store_reflink(src, dst)
|
||||
return {
|
||||
"reflinked": 1 if reflinked else 0,
|
||||
"copied": 0 if reflinked else 1,
|
||||
"failed": 0,
|
||||
"total_bytes": src.stat().st_size if src.exists() else 0,
|
||||
"dst": dst,
|
||||
}
|
||||
|
||||
|
||||
def cas_exists(config, key) -> bool:
|
||||
"""Check if a CAS entry exists."""
|
||||
return Path(cas_path(config, key)).exists()
|
||||
|
||||
|
||||
def cas_retrieve(config, key, dst_path) -> bool:
|
||||
"""Retrieve a CAS entry back to `dst_path` (reflink copy)."""
|
||||
src = Path(cas_path(config, key))
|
||||
if not src.exists():
|
||||
return False
|
||||
if src.is_dir():
|
||||
stats = reflink_snapshot(src, dst_path)
|
||||
return stats["failed"] == 0
|
||||
else:
|
||||
return store_reflink(src, dst_path)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue