159 lines
6.3 KiB
Python
159 lines
6.3 KiB
Python
"""
|
|
Metrics endpoint — Prometheus exposition + JSON view for the UI.
|
|
|
|
Emits metric names matching add-to-grafana-config.json:
|
|
- fester_node_cpu (gauge, per node)
|
|
- fester_node_temp (gauge, per node)
|
|
- fester_node_active_jobs (gauge, per node)
|
|
- fester_node_instability (gauge, per node)
|
|
- fester_nodes_total (gauge)
|
|
- fester_nodes_online (gauge)
|
|
- fester_cluster_avg_heat (gauge)
|
|
- fester_cluster_active_jobs (gauge)
|
|
- fester_pipeline_actions_total (counter, per state) ← grafana expects this
|
|
- fester_cache_hits_total (counter, per target) ← grafana expects this
|
|
- fester_scheduler_score (gauge, per node+target)
|
|
"""
|
|
|
|
from fastapi import APIRouter
|
|
from fastapi.responses import PlainTextResponse
|
|
from typing import Optional
|
|
import threading
|
|
|
|
from backend.metrics.observability import ObservabilityHub
|
|
from backend.metrics.node_state import NodeStateRegistry
|
|
|
|
router = APIRouter(tags=["metrics"])
|
|
|
|
# Process-wide counters (incremented by main.py's bus subscriber)
|
|
_pipeline_action_counts: dict = {} # state -> count
|
|
_cache_hit_counts: dict = {} # target -> count
|
|
_scheduler_scores: dict = {} # (node, target) -> score
|
|
_metrics_lock = threading.Lock()
|
|
|
|
|
|
def record_pipeline_action(state: str):
|
|
"""Called by main.py's bus subscriber when a task_update event fires."""
|
|
with _metrics_lock:
|
|
_pipeline_action_counts[state] = _pipeline_action_counts.get(state, 0) + 1
|
|
|
|
|
|
def record_cache_hit(target: str):
|
|
with _metrics_lock:
|
|
_cache_hit_counts[target] = _cache_hit_counts.get(target, 0) + 1
|
|
|
|
|
|
def record_scheduler_score(node: str, target: str, score: float):
|
|
with _metrics_lock:
|
|
_scheduler_scores[(node, target)] = score
|
|
|
|
|
|
def _gather_metrics(registry: NodeStateRegistry):
|
|
"""Build a JSON-serializable metrics snapshot from the node state registry."""
|
|
nodes = registry.all()
|
|
total_jobs = sum(n.active_jobs for n in nodes)
|
|
avg_heat = (sum(n.temp for n in nodes) / len(nodes)) if nodes else 0.0
|
|
online = sum(1 for n in nodes if n.temp > 0 or n.active_jobs > 0)
|
|
|
|
return {
|
|
"nodes": {
|
|
"total": len(nodes),
|
|
"online": online,
|
|
"offline": len(nodes) - online,
|
|
},
|
|
"cluster": {
|
|
"avg_heat": round(avg_heat, 1),
|
|
"total_active_jobs": total_jobs,
|
|
},
|
|
"per_node": [
|
|
{
|
|
"name": n.name,
|
|
"cpu_load": round(n.cpu_load, 2),
|
|
"memory_load": round(n.memory_load, 2),
|
|
"temp": round(n.temp, 1),
|
|
"instability": round(n.instability, 3),
|
|
"active_jobs": n.active_jobs,
|
|
"last_seen": n.last_seen,
|
|
}
|
|
for n in nodes
|
|
],
|
|
"pipeline_actions": dict(_pipeline_action_counts),
|
|
"cache_hits": dict(_cache_hit_counts),
|
|
}
|
|
|
|
|
|
@router.get("/api/metrics/json")
|
|
def metrics_json():
|
|
"""JSON view of current metrics — consumed by the UI."""
|
|
from backend.api.api import registry
|
|
return _gather_metrics(registry)
|
|
|
|
|
|
@router.get("/metrics", response_class=PlainTextResponse)
|
|
def metrics_prom():
|
|
"""Prometheus exposition format — names match add-to-grafana-config.json."""
|
|
from backend.api.api import registry
|
|
snap = _gather_metrics(registry)
|
|
lines = []
|
|
|
|
# Cluster-level gauges
|
|
lines.append("# HELP fester_nodes_total Total nodes known to the cluster")
|
|
lines.append("# TYPE fester_nodes_total gauge")
|
|
lines.append(f"fester_nodes_total {snap['nodes']['total']}")
|
|
lines.append("# HELP fester_nodes_online Online nodes")
|
|
lines.append("# TYPE fester_nodes_online gauge")
|
|
lines.append(f"fester_nodes_online {snap['nodes']['online']}")
|
|
lines.append("# HELP fester_cluster_avg_heat Average cluster heat")
|
|
lines.append("# TYPE fester_cluster_avg_heat gauge")
|
|
lines.append(f"fester_cluster_avg_heat {snap['cluster']['avg_heat']}")
|
|
lines.append("# HELP fester_cluster_active_jobs Total active jobs")
|
|
lines.append("# TYPE fester_cluster_active_jobs gauge")
|
|
lines.append(f"fester_cluster_active_jobs {snap['cluster']['total_active_jobs']}")
|
|
|
|
# Per-node gauges — emit BOTH names so existing dashboards (which use
|
|
# fester_node_cpu and fester_node_temp) work, and new ones (cpu_load etc)
|
|
# also work.
|
|
for n in snap["per_node"]:
|
|
labels = f'node="{n["name"]}"'
|
|
# Canonical names (matching grafana config)
|
|
lines.append(f'fester_node_cpu{{{labels}}} {n["cpu_load"]}')
|
|
lines.append(f'fester_node_temp{{{labels}}} {n["temp"]}')
|
|
# Additional names
|
|
lines.append(f'fester_node_active_jobs{{{labels}}} {n["active_jobs"]}')
|
|
lines.append(f'fester_node_instability{{{labels}}} {n["instability"]}')
|
|
lines.append(f'fester_node_memory_load{{{labels}}} {n["memory_load"]}')
|
|
|
|
# Pipeline action counter (grafana expects fester_pipeline_actions_total)
|
|
lines.append("# HELP fester_pipeline_actions_total Total pipeline actions by state")
|
|
lines.append("# TYPE fester_pipeline_actions_total counter")
|
|
with _metrics_lock:
|
|
for state, count in _pipeline_action_counts.items():
|
|
lines.append(f'fester_pipeline_actions_total{{state="{state}"}} {count}')
|
|
|
|
# Cache hit counter (grafana expects fester_cache_hits_total)
|
|
lines.append("# HELP fester_cache_hits_total Total cache hits by target")
|
|
lines.append("# TYPE fester_cache_hits_total counter")
|
|
with _metrics_lock:
|
|
for target, count in _cache_hit_counts.items():
|
|
lines.append(f'fester_cache_hits_total{{target="{target}"}} {count}')
|
|
|
|
# Scheduler score gauge (grafana expects fester_scheduler_score)
|
|
lines.append("# HELP fester_scheduler_score Last scheduler score per node+target")
|
|
lines.append("# TYPE fester_scheduler_score gauge")
|
|
with _metrics_lock:
|
|
for (node, target), score in _scheduler_scores.items():
|
|
lines.append(f'fester_scheduler_score{{node="{node}",target="{target}"}} {score}')
|
|
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
# -----------------------------
|
|
# Legacy wrapper for api.py facade
|
|
# -----------------------------
|
|
def metrics_endpoint(registry=None):
|
|
"""Legacy — returns the JSON metrics snapshot."""
|
|
if registry is None:
|
|
from backend.api.api import registry as r
|
|
registry = r
|
|
return _gather_metrics(registry)
|