""" Metrics endpoint — Prometheus exposition + JSON view for the UI. Emits metric names matching add-to-grafana-config.json: - fester_node_cpu (gauge, per node) - fester_node_temp (gauge, per node) - fester_node_active_jobs (gauge, per node) - fester_node_instability (gauge, per node) - fester_nodes_total (gauge) - fester_nodes_online (gauge) - fester_cluster_avg_heat (gauge) - fester_cluster_active_jobs (gauge) - fester_pipeline_actions_total (counter, per state) ← grafana expects this - fester_cache_hits_total (counter, per target) ← grafana expects this - fester_scheduler_score (gauge, per node+target) """ from fastapi import APIRouter from fastapi.responses import PlainTextResponse from typing import Optional import threading from backend.metrics.observability import ObservabilityHub from backend.metrics.node_state import NodeStateRegistry router = APIRouter(tags=["metrics"]) # Process-wide counters (incremented by main.py's bus subscriber) _pipeline_action_counts: dict = {} # state -> count _cache_hit_counts: dict = {} # target -> count _scheduler_scores: dict = {} # (node, target) -> score _metrics_lock = threading.Lock() def record_pipeline_action(state: str): """Called by main.py's bus subscriber when a task_update event fires.""" with _metrics_lock: _pipeline_action_counts[state] = _pipeline_action_counts.get(state, 0) + 1 def record_cache_hit(target: str): with _metrics_lock: _cache_hit_counts[target] = _cache_hit_counts.get(target, 0) + 1 def record_scheduler_score(node: str, target: str, score: float): with _metrics_lock: _scheduler_scores[(node, target)] = score def _gather_metrics(registry: NodeStateRegistry): """Build a JSON-serializable metrics snapshot from the node state registry.""" nodes = registry.all() total_jobs = sum(n.active_jobs for n in nodes) avg_heat = (sum(n.temp for n in nodes) / len(nodes)) if nodes else 0.0 online = sum(1 for n in nodes if n.temp > 0 or n.active_jobs > 0) return { "nodes": { "total": len(nodes), "online": online, "offline": len(nodes) - online, }, "cluster": { "avg_heat": round(avg_heat, 1), "total_active_jobs": total_jobs, }, "per_node": [ { "name": n.name, "cpu_load": round(n.cpu_load, 2), "memory_load": round(n.memory_load, 2), "temp": round(n.temp, 1), "instability": round(n.instability, 3), "active_jobs": n.active_jobs, "last_seen": n.last_seen, } for n in nodes ], "pipeline_actions": dict(_pipeline_action_counts), "cache_hits": dict(_cache_hit_counts), } @router.get("/api/metrics/json") def metrics_json(): """JSON view of current metrics — consumed by the UI.""" from backend.api.api import registry return _gather_metrics(registry) @router.get("/metrics", response_class=PlainTextResponse) def metrics_prom(): """Prometheus exposition format — names match add-to-grafana-config.json.""" from backend.api.api import registry snap = _gather_metrics(registry) lines = [] # Cluster-level gauges lines.append("# HELP fester_nodes_total Total nodes known to the cluster") lines.append("# TYPE fester_nodes_total gauge") lines.append(f"fester_nodes_total {snap['nodes']['total']}") lines.append("# HELP fester_nodes_online Online nodes") lines.append("# TYPE fester_nodes_online gauge") lines.append(f"fester_nodes_online {snap['nodes']['online']}") lines.append("# HELP fester_cluster_avg_heat Average cluster heat") lines.append("# TYPE fester_cluster_avg_heat gauge") lines.append(f"fester_cluster_avg_heat {snap['cluster']['avg_heat']}") lines.append("# HELP fester_cluster_active_jobs Total active jobs") lines.append("# TYPE fester_cluster_active_jobs gauge") lines.append(f"fester_cluster_active_jobs {snap['cluster']['total_active_jobs']}") # Per-node gauges — emit BOTH names so existing dashboards (which use # fester_node_cpu and fester_node_temp) work, and new ones (cpu_load etc) # also work. for n in snap["per_node"]: labels = f'node="{n["name"]}"' # Canonical names (matching grafana config) lines.append(f'fester_node_cpu{{{labels}}} {n["cpu_load"]}') lines.append(f'fester_node_temp{{{labels}}} {n["temp"]}') # Additional names lines.append(f'fester_node_active_jobs{{{labels}}} {n["active_jobs"]}') lines.append(f'fester_node_instability{{{labels}}} {n["instability"]}') lines.append(f'fester_node_memory_load{{{labels}}} {n["memory_load"]}') # Pipeline action counter (grafana expects fester_pipeline_actions_total) lines.append("# HELP fester_pipeline_actions_total Total pipeline actions by state") lines.append("# TYPE fester_pipeline_actions_total counter") with _metrics_lock: for state, count in _pipeline_action_counts.items(): lines.append(f'fester_pipeline_actions_total{{state="{state}"}} {count}') # Cache hit counter (grafana expects fester_cache_hits_total) lines.append("# HELP fester_cache_hits_total Total cache hits by target") lines.append("# TYPE fester_cache_hits_total counter") with _metrics_lock: for target, count in _cache_hit_counts.items(): lines.append(f'fester_cache_hits_total{{target="{target}"}} {count}') # Scheduler score gauge (grafana expects fester_scheduler_score) lines.append("# HELP fester_scheduler_score Last scheduler score per node+target") lines.append("# TYPE fester_scheduler_score gauge") with _metrics_lock: for (node, target), score in _scheduler_scores.items(): lines.append(f'fester_scheduler_score{{node="{node}",target="{target}"}} {score}') return "\n".join(lines) + "\n" # ----------------------------- # Legacy wrapper for api.py facade # ----------------------------- def metrics_endpoint(registry=None): """Legacy — returns the JSON metrics snapshot.""" if registry is None: from backend.api.api import registry as r registry = r return _gather_metrics(registry)