1175 lines
29 KiB
JSON
Executable File
1175 lines
29 KiB
JSON
Executable File
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "timeseries",
|
|
"title": "Request Volume (req/s)",
|
|
"description": "Per-second HTTP request rate from frosty_requests_total, broken out by route and status.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (route, status) (rate(frosty_requests_total[$__rate_interval]))",
|
|
"legendFormat": "{{route}} {{status}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "timeseries",
|
|
"title": "Request Latency percentiles (ms)",
|
|
"description": "frosty_request_duration_ms summary, plotted by pre-computed quantile (p50 / p95 / p99) per route.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"max",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "frosty_request_duration_ms{quantile=\"0.50\"}",
|
|
"legendFormat": "{{route}} p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "frosty_request_duration_ms{quantile=\"0.95\"}",
|
|
"legendFormat": "{{route}} p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "frosty_request_duration_ms{quantile=\"0.99\"}",
|
|
"legendFormat": "{{route}} p99"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "Input vs Output Tokens by Model (tokens/s)",
|
|
"description": "Token throughput: rate of frosty_input_tokens_total and frosty_output_tokens_total, summed per model.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (model) (rate(frosty_input_tokens_total[$__rate_interval]))",
|
|
"legendFormat": "in {{model}}"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (model) (rate(frosty_output_tokens_total[$__rate_interval]))",
|
|
"legendFormat": "out {{model}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "timeseries",
|
|
"title": "LLM Cost Rate by Model (USD/s)",
|
|
"description": "Spend velocity in USD per second, from the rate of the frosty_llm_cost_usd_total counter, per provider and model.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "currencyUSD",
|
|
"decimals": 6,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 15,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (provider, model) (rate(frosty_llm_cost_usd_total[$__rate_interval]))",
|
|
"legendFormat": "{{provider}}/{{model}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "Total Cost (USD)",
|
|
"description": "Cumulative global spend since process start, from the frosty_cost_usd_total counter.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "currencyUSD",
|
|
"decimals": 4,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "frosty_cost_usd_total",
|
|
"legendFormat": "total",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "gauge",
|
|
"title": "Cache Hit Ratio",
|
|
"description": "hit / (hit + miss) over the current rate window, from frosty_cache_events_total. Shows No data when there is no cache traffic.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 16
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 0.8
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"orientation": "auto",
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(rate(frosty_cache_events_total{result=\"hit\"}[$__rate_interval])) / sum(rate(frosty_cache_events_total[$__rate_interval]))",
|
|
"legendFormat": "hit ratio",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "LLM Requests by Status Class (req/s)",
|
|
"description": "Success vs error mix: rate of frosty_llm_requests_total summed by status_class (for example 2xx / 4xx / 5xx).",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 16
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 25,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (status_class) (rate(frosty_llm_requests_total[$__rate_interval]))",
|
|
"legendFormat": "{{status_class}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "timeseries",
|
|
"title": "Cache Events by Result (events/s)",
|
|
"description": "Cache hit vs miss throughput: rate of frosty_cache_events_total summed by result.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 15,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (result) (rate(frosty_cache_events_total[$__rate_interval]))",
|
|
"legendFormat": "{{result}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "row",
|
|
"title": "Streaming latency (frosty_stream_*_latency_ms)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "timeseries",
|
|
"title": "Time to First Token (ms)",
|
|
"description": "Percentiles from the frosty_stream_first_token_latency_ms histogram - how long a streamed response takes to emit its first token. This histogram had no panel before.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "rate(frosty_stream_first_token_latency_ms_sum[$__rate_interval]) / rate(frosty_stream_first_token_latency_ms_count[$__rate_interval])",
|
|
"legendFormat": "mean"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "timeseries",
|
|
"title": "Inter-Token Latency (ms)",
|
|
"description": "Percentiles from the frosty_stream_inter_token_latency_ms histogram - the gap between successive streamed tokens. This histogram had no panel before.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 33
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(frosty_stream_inter_token_latency_ms_bucket[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(frosty_stream_inter_token_latency_ms_bucket[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(frosty_stream_inter_token_latency_ms_bucket[$__rate_interval])))",
|
|
"legendFormat": "p99"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "rate(frosty_stream_inter_token_latency_ms_sum[$__rate_interval]) / rate(frosty_stream_inter_token_latency_ms_count[$__rate_interval])",
|
|
"legendFormat": "mean"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "heatmap",
|
|
"title": "TTFT distribution (heatmap)",
|
|
"description": "Full bucket distribution of frosty_stream_first_token_latency_ms, not just percentiles - shows bimodality that a p95 line hides.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 1,
|
|
"color": {
|
|
"mode": "scheme",
|
|
"scheme": "Turbo",
|
|
"steps": 64,
|
|
"reverse": false,
|
|
"exponent": 0.5,
|
|
"fill": "dark-orange"
|
|
},
|
|
"yAxis": {
|
|
"unit": "ms",
|
|
"axisPlacement": "left"
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"cellValues": {},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 1e-09
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval]))",
|
|
"format": "heatmap",
|
|
"legendFormat": "{{le}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "timeseries",
|
|
"title": "Streamed responses (streams/s)",
|
|
"description": "Rate of completed streams from each histogram's _count series. Confirms whether the latency panels above have traffic behind them.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 41
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "rate(frosty_stream_first_token_latency_ms_count[$__rate_interval])",
|
|
"legendFormat": "streams (first token)"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "rate(frosty_stream_inter_token_latency_ms_count[$__rate_interval])",
|
|
"legendFormat": "token gaps"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 14,
|
|
"type": "row",
|
|
"title": "Request throughput and internal counters",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 49
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 15,
|
|
"type": "timeseries",
|
|
"title": "Request Rate by Route (req/s)",
|
|
"description": "Derived from frosty_request_duration_ms_count, the summary's observation count. Complements the status-labelled frosty_requests_total panel and was previously unused.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 50
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (route) (rate(frosty_request_duration_ms_count[$__rate_interval]))",
|
|
"legendFormat": "{{route}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 16,
|
|
"type": "timeseries",
|
|
"title": "Internal Counters (events/s)",
|
|
"description": "frosty_counter{name} is the gateway's generic named-counter series (plugin events, governance denials, fallback reroutes, anything registered at runtime). It had no panel at all, so those events were invisible.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 50
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "cps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (name) (rate(frosty_counter[$__rate_interval]))",
|
|
"legendFormat": "{{name}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 17,
|
|
"type": "table",
|
|
"title": "Internal Counters (totals)",
|
|
"description": "Cumulative value of every frosty_counter series, so counters that fire rarely are still visible.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 58
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"desc": true,
|
|
"displayName": "Value"
|
|
}
|
|
]
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (name) (frosty_counter)",
|
|
"format": "table",
|
|
"instant": true,
|
|
"legendFormat": "{{name}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 18,
|
|
"type": "timeseries",
|
|
"title": "Token Throughput Total (tokens/s)",
|
|
"description": "Aggregate input and output token rate across every provider and model, from frosty_input_tokens_total / frosty_output_tokens_total.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 58
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(rate(frosty_input_tokens_total[$__rate_interval]))",
|
|
"legendFormat": "input"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(rate(frosty_output_tokens_total[$__rate_interval]))",
|
|
"legendFormat": "output"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 39,
|
|
"tags": [
|
|
"frosty",
|
|
"gateway",
|
|
"llm",
|
|
"prometheus"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "Prometheus",
|
|
"value": "prometheus"
|
|
},
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Datasource",
|
|
"multi": false,
|
|
"name": "DS_PROMETHEUS",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"queryValue": "",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {},
|
|
"timezone": "",
|
|
"title": "Frosty Gateway Observability",
|
|
"uid": "frosty-gateway-observability",
|
|
"version": 2,
|
|
"weekStart": ""
|
|
}
|