578 lines
17 KiB
Python
Executable File
578 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Fester CLI — command-line interface to the Fester backend.
|
|
|
|
All commands hit the REST API at FESTER_API (default: http://localhost:8080).
|
|
|
|
Subcommands:
|
|
stream — live event stream (WebSocket)
|
|
status — pipeline state
|
|
sessions — list replay sessions
|
|
|
|
build — kick off a build
|
|
builds — list recent builds
|
|
build-info <id> — show one build's details
|
|
cancel <id> — cancel a running build
|
|
|
|
release — kick off a release build
|
|
targets — list known build targets
|
|
toggle-target <n> [on|off] — enable/disable a target
|
|
|
|
node list — list cluster nodes
|
|
node set-policy <node> <preferred|avoid|neutral> — set node policy
|
|
node probe <node> — manually probe one node's agent
|
|
node probe-all — probe every node in parallel
|
|
|
|
retry <action> — retry a failed action
|
|
force <action> <node> — force an action onto a specific node
|
|
pause — pause the running pipeline
|
|
resume — resume the running pipeline
|
|
|
|
cause explain <node> [action] — explain causal chain for a node
|
|
cause trace — dump the entire cause graph
|
|
cause events [N] — show last N cause events
|
|
blast <action> — compute blast radius if action fails
|
|
|
|
timeline <node> — show events for a node
|
|
rewind <index> — show snapshot at event index
|
|
|
|
autopsy <session_id> <action> — run failure autopsy
|
|
replay start — start a new replay session
|
|
replay step <sid> — step forward
|
|
replay reset <sid> — reset to start
|
|
|
|
debugger state — show current debugger state
|
|
debugger pause — pause running engine
|
|
debugger resume — resume
|
|
debugger step — step one action
|
|
|
|
policy-set <key> <value> — set a policy override
|
|
policy-clear <key> — clear a policy override
|
|
policy-list — list all overrides
|
|
|
|
health — backend health check
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import requests
|
|
|
|
try:
|
|
import websocket
|
|
HAVE_WS = True
|
|
except ImportError:
|
|
HAVE_WS = False
|
|
|
|
|
|
API = os.environ.get("FESTER_API", "http://localhost:8080")
|
|
WS = os.environ.get("FESTER_WS", API.replace("http", "ws") + "/ws")
|
|
|
|
|
|
# -----------------------------
|
|
# HTTP HELPERS
|
|
# -----------------------------
|
|
def get(path):
|
|
try:
|
|
r = requests.get(f"{API}{path}", timeout=10)
|
|
if r.status_code >= 400:
|
|
print(f"error: {r.status_code} {r.text[:200]}", file=sys.stderr)
|
|
sys.exit(1)
|
|
ct = r.headers.get("content-type", "")
|
|
return r.json() if "json" in ct else r.text
|
|
except requests.exceptions.ConnectionError:
|
|
print(f"error: cannot connect to {API} (is the backend running?)", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
|
|
def post(path, body=None):
|
|
try:
|
|
r = requests.post(f"{API}{path}", json=body or {}, timeout=10)
|
|
if r.status_code >= 400:
|
|
print(f"error: {r.status_code} {r.text[:200]}", file=sys.stderr)
|
|
sys.exit(1)
|
|
ct = r.headers.get("content-type", "")
|
|
return r.json() if "json" in ct else r.text
|
|
except requests.exceptions.ConnectionError:
|
|
print(f"error: cannot connect to {API} (is the backend running?)", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
|
|
def pp(obj):
|
|
"""Pretty-print a JSON object."""
|
|
if isinstance(obj, (dict, list)):
|
|
print(json.dumps(obj, indent=2))
|
|
else:
|
|
print(obj)
|
|
|
|
|
|
# -----------------------------
|
|
# LIVE STREAM
|
|
# -----------------------------
|
|
def stream():
|
|
if not HAVE_WS:
|
|
print("error: pip install websocket-client", file=sys.stderr)
|
|
sys.exit(1)
|
|
ws = websocket.WebSocket()
|
|
ws.connect(WS)
|
|
print(f"📡 Fester event stream active → {WS}\n", file=sys.stderr)
|
|
while True:
|
|
msg = ws.recv()
|
|
try:
|
|
e = json.loads(msg)
|
|
except Exception:
|
|
print(msg)
|
|
continue
|
|
ts = e.get("timestamp", 0)
|
|
t = time.strftime("%H:%M:%S", time.localtime(ts)) if ts else "--:--:--"
|
|
etype = e.get("type", "event")
|
|
action = e.get("action") or ""
|
|
node = e.get("node") or ""
|
|
state = e.get("state") or ""
|
|
meta = e.get("meta") or {}
|
|
extras = []
|
|
if meta.get("critical"):
|
|
extras.append("⚠ critical")
|
|
if meta.get("cache"):
|
|
extras.append(f"cache={meta['cache']}")
|
|
if meta.get("target"):
|
|
extras.append(f"target={meta['target']}")
|
|
if meta.get("build_id"):
|
|
extras.append(f"build={meta['build_id']}")
|
|
if e.get("reason"):
|
|
extras.append(f"reason={e['reason']}")
|
|
extra_str = " ".join(extras)
|
|
print(f"[{t}] {etype:14s} {action:25s} {node:15s} {state:10s} {extra_str}")
|
|
|
|
|
|
# -----------------------------
|
|
# BUILD COMMANDS
|
|
# -----------------------------
|
|
def build(args):
|
|
body = {
|
|
"cmd": args.cmd,
|
|
"dir": args.dir,
|
|
"project": args.project,
|
|
"arch": args.arch,
|
|
"target": args.target,
|
|
"toolchain": args.toolchain,
|
|
}
|
|
res = post("/api/build", body)
|
|
pp(res)
|
|
if args.watch:
|
|
print(f"\n📡 Watching events for build {res.get('build_id')}... (Ctrl+C to stop)",
|
|
file=sys.stderr)
|
|
if HAVE_WS:
|
|
ws = websocket.WebSocket()
|
|
ws.connect(WS)
|
|
try:
|
|
while True:
|
|
msg = ws.recv()
|
|
e = json.loads(msg)
|
|
if e.get("meta", {}).get("build_id") == res.get("build_id") or \
|
|
e.get("build_id") == res.get("build_id"):
|
|
print(f" {e.get('type'):14s} {e.get('action', ''):25s} → {e.get('state', '')}")
|
|
except KeyboardInterrupt:
|
|
print("\n(stopped)", file=sys.stderr)
|
|
|
|
|
|
def builds(args):
|
|
res = get("/api/builds")
|
|
if args.json:
|
|
pp(res)
|
|
return
|
|
b = res.get("builds", [])
|
|
if not b:
|
|
print("(no builds)")
|
|
return
|
|
print(f"{'BUILD_ID':10s} {'STATUS':10s} {'CMD':25s} {'STARTED':15s} {'DUR':6s}")
|
|
for x in b:
|
|
dur = f"{x['ended_at'] - x['started_at']:.1f}s" if x.get("ended_at") else "—"
|
|
print(f"{x['build_id']:10s} {x['status']:10s} {(x.get('cmd') or '')[:25]:25s} "
|
|
f"{time.strftime('%Y-%m-%d %H:%M', time.localtime(x['started_at']))} {dur}")
|
|
|
|
|
|
def build_info(args):
|
|
pp(get(f"/api/builds/{args.build_id}"))
|
|
|
|
|
|
def cancel(args):
|
|
pp(post(f"/api/builds/{args.build_id}/cancel"))
|
|
|
|
|
|
def release(args):
|
|
body = {"repo": args.repo, "project": args.project, "target": args.target}
|
|
pp(post("/api/release", body))
|
|
|
|
|
|
# -----------------------------
|
|
# TARGETS
|
|
# -----------------------------
|
|
def targets(args):
|
|
res = get("/api/targets")
|
|
if args.json:
|
|
pp(res)
|
|
return
|
|
print(f"{'NAME':25s} {'ENABLED':8s} {'TYPE':15s} {'ARCH':10s}")
|
|
for t in res:
|
|
print(f"{t['name']:25s} {'✓' if t.get('enabled') else '✗':8s} "
|
|
f"{t.get('type', '—'):15s} {t.get('arch', '—'):10s}")
|
|
|
|
|
|
def toggle_target(args):
|
|
enabled = args.state.lower() in ("on", "true", "1", "yes")
|
|
pp(post(f"/api/targets/{args.name}", {"enabled": enabled}))
|
|
|
|
|
|
# -----------------------------
|
|
# NODES
|
|
# -----------------------------
|
|
def node_list(args):
|
|
res = get("/api/nodes")
|
|
if args.json:
|
|
pp(res)
|
|
return
|
|
n = res.get("nodes", [])
|
|
print(f"{'NAME':15s} {'HOST':18s} {'STATE':8s} {'POLICY':10s} {'HEAT':5s} {'JOBS':8s}")
|
|
for x in n:
|
|
print(f"{x['name']:15s} {x.get('host', '—'):18s} {x.get('state', '—'):8s} "
|
|
f"{x.get('policy', '—'):10s} {str(round(x.get('heat', 0))) + '°':5s} "
|
|
f"{str(x.get('jobs', 0)) + '/' + str(x.get('max_jobs', 0)):8s}")
|
|
|
|
|
|
def node_set_policy(args):
|
|
pp(post(f"/api/nodes/{args.node}/policy", {"policy": args.policy}))
|
|
|
|
|
|
def node_probe(args):
|
|
pp(post(f"/api/nodes/{args.node}/probe"))
|
|
|
|
|
|
def node_probe_all(args):
|
|
pp(post("/api/nodes/probe-all"))
|
|
|
|
|
|
# -----------------------------
|
|
# PIPELINE CONTROL
|
|
# -----------------------------
|
|
def status(args):
|
|
pp(get("/api/pipeline/state"))
|
|
|
|
|
|
def retry(args):
|
|
pp(post("/api/pipeline/retry", {"action": args.action}))
|
|
|
|
|
|
def force(args):
|
|
pp(post("/api/pipeline/force-node", {"action": args.action, "node": args.node}))
|
|
|
|
|
|
def pause(args):
|
|
pp(post("/api/pipeline/pause"))
|
|
|
|
|
|
def resume(args):
|
|
pp(post("/api/pipeline/resume"))
|
|
|
|
|
|
# -----------------------------
|
|
# SESSIONS
|
|
# -----------------------------
|
|
def sessions(args):
|
|
res = get("/api/sessions")
|
|
if args.json:
|
|
pp(res)
|
|
return
|
|
s = res.get("sessions", [])
|
|
if not s:
|
|
print("(no sessions)")
|
|
return
|
|
print(f"{'ID':20s} {'SOURCE':15s} {'EVENTS':7s} {'CREATED':15s}")
|
|
for x in s:
|
|
print(f"{x['id'][:20]:20s} {x.get('source', '—'):15s} {str(x.get('events', 0)):7s} "
|
|
f"{time.strftime('%Y-%m-%d %H:%M', time.localtime(x.get('created_at', 0)))}")
|
|
|
|
|
|
def attach(args):
|
|
pp(post("/api/session/attach", {"action": args.action}))
|
|
|
|
|
|
def detach(args):
|
|
pp(post("/api/session/detach", {"action": args.action}))
|
|
|
|
|
|
# -----------------------------
|
|
# CAUSE
|
|
# -----------------------------
|
|
def cause_explain(args):
|
|
path = f"/api/cause/explain/{args.node}"
|
|
if args.action:
|
|
path += f"?action={args.action}"
|
|
pp(get(path))
|
|
|
|
|
|
def cause_trace(args):
|
|
pp(get("/api/cause/trace"))
|
|
|
|
|
|
def cause_events(args):
|
|
pp(get(f"/api/cause/events?limit={args.limit}"))
|
|
|
|
|
|
def blast(args):
|
|
pp(get(f"/api/propagation/{args.action}"))
|
|
|
|
|
|
# -----------------------------
|
|
# TIMELINE
|
|
# -----------------------------
|
|
def timeline(args):
|
|
pp(get(f"/timeline/{args.node}"))
|
|
|
|
|
|
def rewind(args):
|
|
pp(get(f"/timeline/rewind/{args.index}"))
|
|
|
|
|
|
# -----------------------------
|
|
# AUTOPSY + REPLAY
|
|
# -----------------------------
|
|
def autopsy(args):
|
|
pp(get(f"/autopsy/{args.session_id}/{args.action}"))
|
|
|
|
|
|
def replay_start(args):
|
|
pp(post("/replay/start", {"journal": args.journal or "live"}))
|
|
|
|
|
|
def replay_step(args):
|
|
pp(post(f"/replay/step?session_id={args.session_id}"))
|
|
|
|
|
|
def replay_reset(args):
|
|
pp(post(f"/replay/reset?session_id={args.session_id}"))
|
|
|
|
|
|
# -----------------------------
|
|
# DEBUGGER
|
|
# -----------------------------
|
|
def debugger_state(args):
|
|
pp(get("/debugger/state"))
|
|
|
|
|
|
def debugger_pause(args):
|
|
pp(post("/debugger/pause"))
|
|
|
|
|
|
def debugger_resume(args):
|
|
pp(post("/debugger/resume"))
|
|
|
|
|
|
def debugger_step(args):
|
|
pp(post("/debugger/step"))
|
|
|
|
|
|
# -----------------------------
|
|
# POLICY
|
|
# -----------------------------
|
|
def policy_set(args):
|
|
pp(post("/api/policy/set", {"key": args.key, "value": args.value}))
|
|
|
|
|
|
def policy_clear(args):
|
|
pp(post("/api/policy/clear", {"key": args.key}))
|
|
|
|
|
|
def policy_list(args):
|
|
pp(get("/api/policy"))
|
|
|
|
|
|
# -----------------------------
|
|
# HEALTH
|
|
# -----------------------------
|
|
def health(args):
|
|
pp(get("/api/health"))
|
|
|
|
|
|
# -----------------------------
|
|
# CLI ENTRYPOINT
|
|
# -----------------------------
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
prog="fester",
|
|
description="Fester distributed build system CLI",
|
|
epilog=f"Backend: {API} (override with FESTER_API env var)",
|
|
)
|
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
|
|
# stream
|
|
sub.add_parser("stream", help="live event stream (WebSocket)")
|
|
|
|
# build
|
|
p = sub.add_parser("build", help="kick off a build")
|
|
p.add_argument("--cmd", default="make -j$(nproc)")
|
|
p.add_argument("--dir", default="/home/user/linux")
|
|
p.add_argument("--project", default=None)
|
|
p.add_argument("--arch", default="x86_64")
|
|
p.add_argument("--target", default="linux-gnu")
|
|
p.add_argument("--toolchain", default="gcc")
|
|
p.add_argument("--watch", action="store_true", help="watch events after starting")
|
|
p.set_defaults(func=build)
|
|
|
|
p = sub.add_parser("builds", help="list recent builds")
|
|
p.add_argument("--json", action="store_true")
|
|
p.set_defaults(func=builds)
|
|
|
|
p = sub.add_parser("build-info", help="show one build's details")
|
|
p.add_argument("build_id")
|
|
p.set_defaults(func=build_info)
|
|
|
|
p = sub.add_parser("cancel", help="cancel a running build")
|
|
p.add_argument("build_id")
|
|
p.set_defaults(func=cancel)
|
|
|
|
# release
|
|
p = sub.add_parser("release", help="kick off a release build")
|
|
p.add_argument("--repo", default="https://forgejo.local/project.git")
|
|
p.add_argument("--project", default=None)
|
|
p.add_argument("--target", default="linux-gnu")
|
|
p.set_defaults(func=release)
|
|
|
|
# targets
|
|
p = sub.add_parser("targets", help="list known build targets")
|
|
p.add_argument("--json", action="store_true")
|
|
p.set_defaults(func=targets)
|
|
|
|
p = sub.add_parser("toggle-target", help="enable/disable a target")
|
|
p.add_argument("name")
|
|
p.add_argument("state", choices=["on", "off", "true", "false", "1", "0", "yes", "no"])
|
|
p.set_defaults(func=toggle_target)
|
|
|
|
# node (nested)
|
|
p_node = sub.add_parser("node", help="node management")
|
|
node_sub = p_node.add_subparsers(dest="node_cmd", required=True)
|
|
|
|
p = node_sub.add_parser("list", help="list cluster nodes")
|
|
p.add_argument("--json", action="store_true")
|
|
p.set_defaults(func=node_list)
|
|
|
|
p = node_sub.add_parser("set-policy", help="set node policy")
|
|
p.add_argument("node")
|
|
p.add_argument("policy", choices=["preferred", "avoid", "neutral"])
|
|
p.set_defaults(func=node_set_policy)
|
|
|
|
p = node_sub.add_parser("probe", help="manually probe one node")
|
|
p.add_argument("node")
|
|
p.set_defaults(func=node_probe)
|
|
|
|
p = node_sub.add_parser("probe-all", help="probe all nodes in parallel")
|
|
p.set_defaults(func=node_probe_all)
|
|
|
|
# pipeline control
|
|
p = sub.add_parser("status", help="pipeline state")
|
|
p.set_defaults(func=status)
|
|
p = sub.add_parser("retry", help="retry an action")
|
|
p.add_argument("action")
|
|
p.set_defaults(func=retry)
|
|
p = sub.add_parser("force", help="force an action onto a node")
|
|
p.add_argument("action")
|
|
p.add_argument("node")
|
|
p.set_defaults(func=force)
|
|
p = sub.add_parser("pause", help="pause pipeline")
|
|
p.set_defaults(func=pause)
|
|
p = sub.add_parser("resume", help="resume pipeline")
|
|
p.set_defaults(func=resume)
|
|
|
|
# sessions
|
|
p = sub.add_parser("sessions", help="list replay sessions")
|
|
p.add_argument("--json", action="store_true")
|
|
p.set_defaults(func=sessions)
|
|
p = sub.add_parser("attach", help="attach to a session (IDE feature)")
|
|
p.add_argument("action")
|
|
p.set_defaults(func=attach)
|
|
p = sub.add_parser("detach", help="detach from a session")
|
|
p.add_argument("action")
|
|
p.set_defaults(func=detach)
|
|
|
|
# cause
|
|
p_cause = sub.add_parser("cause", help="cause graph queries")
|
|
cause_sub = p_cause.add_subparsers(dest="cause_cmd", required=True)
|
|
p = cause_sub.add_parser("explain", help="explain causal chain for a node")
|
|
p.add_argument("node")
|
|
p.add_argument("action", nargs="?", default=None)
|
|
p.set_defaults(func=cause_explain)
|
|
p = cause_sub.add_parser("trace", help="full cause graph dump")
|
|
p.set_defaults(func=cause_trace)
|
|
p = cause_sub.add_parser("events", help="recent cause events")
|
|
p.add_argument("limit", type=int, nargs="?", default=50)
|
|
p.set_defaults(func=cause_events)
|
|
|
|
# blast radius
|
|
p = sub.add_parser("blast", help="compute blast radius if action fails")
|
|
p.add_argument("action")
|
|
p.set_defaults(func=blast)
|
|
|
|
# timeline
|
|
p = sub.add_parser("timeline", help="events for a node")
|
|
p.add_argument("node")
|
|
p.set_defaults(func=timeline)
|
|
p = sub.add_parser("rewind", help="snapshot at event index")
|
|
p.add_argument("index", type=int)
|
|
p.set_defaults(func=rewind)
|
|
|
|
# autopsy
|
|
p = sub.add_parser("autopsy", help="run failure autopsy")
|
|
p.add_argument("session_id")
|
|
p.add_argument("action")
|
|
p.set_defaults(func=autopsy)
|
|
|
|
# replay (nested)
|
|
p_replay = sub.add_parser("replay", help="replay session control")
|
|
replay_sub = p_replay.add_subparsers(dest="replay_cmd", required=True)
|
|
p = replay_sub.add_parser("start", help="start a new replay session")
|
|
p.add_argument("--journal", default="live")
|
|
p.set_defaults(func=replay_start)
|
|
p = replay_sub.add_parser("step", help="step forward")
|
|
p.add_argument("session_id")
|
|
p.set_defaults(func=replay_step)
|
|
p = replay_sub.add_parser("reset", help="reset to start")
|
|
p.add_argument("session_id")
|
|
p.set_defaults(func=replay_reset)
|
|
|
|
# debugger (nested)
|
|
p_dbg = sub.add_parser("debugger", help="debugger control")
|
|
dbg_sub = p_dbg.add_subparsers(dest="dbg_cmd", required=True)
|
|
p = dbg_sub.add_parser("state", help="show debugger state")
|
|
p.set_defaults(func=debugger_state)
|
|
p = dbg_sub.add_parser("pause", help="pause running engine")
|
|
p.set_defaults(func=debugger_pause)
|
|
p = dbg_sub.add_parser("resume", help="resume")
|
|
p.set_defaults(func=debugger_resume)
|
|
p = dbg_sub.add_parser("step", help="step one action")
|
|
p.set_defaults(func=debugger_step)
|
|
|
|
# policy
|
|
p = sub.add_parser("policy-set", help="set a policy override")
|
|
p.add_argument("key")
|
|
p.add_argument("value")
|
|
p.set_defaults(func=policy_set)
|
|
p = sub.add_parser("policy-clear", help="clear a policy override")
|
|
p.add_argument("key")
|
|
p.set_defaults(func=policy_clear)
|
|
p = sub.add_parser("policy-list", help="list all overrides")
|
|
p.set_defaults(func=policy_list)
|
|
|
|
# health
|
|
p = sub.add_parser("health", help="backend health check")
|
|
p.set_defaults(func=health)
|
|
|
|
args = parser.parse_args()
|
|
args.func(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|