{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": false, "panels": [ { "id": 1, "type": "timeseries", "title": "Request Volume (req/s)", "description": "Per-second HTTP request rate from frosty_requests_total, broken out by route and status.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (route, status) (rate(frosty_requests_total[$__rate_interval]))", "legendFormat": "{{route}} {{status}}" } ] }, { "id": 2, "type": "timeseries", "title": "Request Latency percentiles (ms)", "description": "frosty_request_duration_ms summary, plotted by pre-computed quantile (p50 / p95 / p99) per route.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "fieldConfig": { "defaults": { "unit": "ms", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 0, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "max", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "frosty_request_duration_ms{quantile=\"0.50\"}", "legendFormat": "{{route}} p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "frosty_request_duration_ms{quantile=\"0.95\"}", "legendFormat": "{{route}} p95" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "frosty_request_duration_ms{quantile=\"0.99\"}", "legendFormat": "{{route}} p99" } ] }, { "id": 3, "type": "timeseries", "title": "Input vs Output Tokens by Model (tokens/s)", "description": "Token throughput: rate of frosty_input_tokens_total and frosty_output_tokens_total, summed per model.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (model) (rate(frosty_input_tokens_total[$__rate_interval]))", "legendFormat": "in {{model}}" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (model) (rate(frosty_output_tokens_total[$__rate_interval]))", "legendFormat": "out {{model}}" } ] }, { "id": 4, "type": "timeseries", "title": "LLM Cost Rate by Model (USD/s)", "description": "Spend velocity in USD per second, from the rate of the frosty_llm_cost_usd_total counter, per provider and model.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 6, "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (provider, model) (rate(frosty_llm_cost_usd_total[$__rate_interval]))", "legendFormat": "{{provider}}/{{model}}" } ] }, { "id": 5, "type": "stat", "title": "Total Cost (USD)", "description": "Cumulative global spend since process start, from the frosty_cost_usd_total counter.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "fixed", "fixedColor": "green" }, "mappings": [] }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "textMode": "auto" }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "frosty_cost_usd_total", "legendFormat": "total", "instant": true } ] }, { "id": 6, "type": "gauge", "title": "Cache Hit Ratio", "description": "hit / (hit + miss) over the current rate window, from frosty_cache_events_total. Shows No data when there is no cache traffic.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "decimals": 1, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(frosty_cache_events_total{result=\"hit\"}[$__rate_interval])) / sum(rate(frosty_cache_events_total[$__rate_interval]))", "legendFormat": "hit ratio", "instant": true } ] }, { "id": 7, "type": "timeseries", "title": "LLM Requests by Status Class (req/s)", "description": "Success vs error mix: rate of frosty_llm_requests_total summed by status_class (for example 2xx / 4xx / 5xx).", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 25, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (status_class) (rate(frosty_llm_requests_total[$__rate_interval]))", "legendFormat": "{{status_class}}" } ] }, { "id": 8, "type": "timeseries", "title": "Cache Events by Result (events/s)", "description": "Cache hit vs miss throughput: rate of frosty_cache_events_total summed by result.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (result) (rate(frosty_cache_events_total[$__rate_interval]))", "legendFormat": "{{result}}" } ] }, { "id": 9, "type": "row", "title": "Streaming latency (frosty_stream_*_latency_ms)", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "panels": [] }, { "id": 10, "type": "timeseries", "title": "Time to First Token (ms)", "description": "Percentiles from the frosty_stream_first_token_latency_ms histogram - how long a streamed response takes to emit its first token. This histogram had no panel before.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "fieldConfig": { "defaults": { "unit": "ms", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval])))", "legendFormat": "p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval])))", "legendFormat": "p95" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval])))", "legendFormat": "p99" }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(frosty_stream_first_token_latency_ms_sum[$__rate_interval]) / rate(frosty_stream_first_token_latency_ms_count[$__rate_interval])", "legendFormat": "mean" } ] }, { "id": 11, "type": "timeseries", "title": "Inter-Token Latency (ms)", "description": "Percentiles from the frosty_stream_inter_token_latency_ms histogram - the gap between successive streamed tokens. This histogram had no panel before.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, "fieldConfig": { "defaults": { "unit": "ms", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(frosty_stream_inter_token_latency_ms_bucket[$__rate_interval])))", "legendFormat": "p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(frosty_stream_inter_token_latency_ms_bucket[$__rate_interval])))", "legendFormat": "p95" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(frosty_stream_inter_token_latency_ms_bucket[$__rate_interval])))", "legendFormat": "p99" }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(frosty_stream_inter_token_latency_ms_sum[$__rate_interval]) / rate(frosty_stream_inter_token_latency_ms_count[$__rate_interval])", "legendFormat": "mean" } ] }, { "id": 12, "type": "heatmap", "title": "TTFT distribution (heatmap)", "description": "Full bucket distribution of frosty_stream_first_token_latency_ms, not just percentiles - shows bimodality that a p95 line hides.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "options": { "calculate": false, "cellGap": 1, "color": { "mode": "scheme", "scheme": "Turbo", "steps": 64, "reverse": false, "exponent": 0.5, "fill": "dark-orange" }, "yAxis": { "unit": "ms", "axisPlacement": "left" }, "legend": { "show": true }, "tooltip": { "show": true, "yHistogram": true }, "rowsFrame": { "layout": "auto" }, "cellValues": {}, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (le) (rate(frosty_stream_first_token_latency_ms_bucket[$__rate_interval]))", "format": "heatmap", "legendFormat": "{{le}}" } ] }, { "id": 13, "type": "timeseries", "title": "Streamed responses (streams/s)", "description": "Rate of completed streams from each histogram's _count series. Confirms whether the latency panels above have traffic behind them.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(frosty_stream_first_token_latency_ms_count[$__rate_interval])", "legendFormat": "streams (first token)" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(frosty_stream_inter_token_latency_ms_count[$__rate_interval])", "legendFormat": "token gaps" } ] }, { "id": 14, "type": "row", "title": "Request throughput and internal counters", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 }, "panels": [] }, { "id": 15, "type": "timeseries", "title": "Request Rate by Route (req/s)", "description": "Derived from frosty_request_duration_ms_count, the summary's observation count. Complements the status-labelled frosty_requests_total panel and was previously unused.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (route) (rate(frosty_request_duration_ms_count[$__rate_interval]))", "legendFormat": "{{route}}" } ] }, { "id": 16, "type": "timeseries", "title": "Internal Counters (events/s)", "description": "frosty_counter{name} is the gateway's generic named-counter series (plugin events, governance denials, fallback reroutes, anything registered at runtime). It had no panel at all, so those events were invisible.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 }, "fieldConfig": { "defaults": { "unit": "cps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (name) (rate(frosty_counter[$__rate_interval]))", "legendFormat": "{{name}}" } ] }, { "id": 17, "type": "table", "title": "Internal Counters (totals)", "description": "Cumulative value of every frosty_counter series, so counters that fire rarely are still visible.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 58 }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true } }, "overrides": [] }, "options": { "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (name) (frosty_counter)", "format": "table", "instant": true, "legendFormat": "{{name}}" } ] }, { "id": 18, "type": "timeseries", "title": "Token Throughput Total (tokens/s)", "description": "Aggregate input and output token rate across every provider and model, from frosty_input_tokens_total / frosty_output_tokens_total.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 58 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(frosty_input_tokens_total[$__rate_interval]))", "legendFormat": "input" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(frosty_output_tokens_total[$__rate_interval]))", "legendFormat": "output" } ] } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "frosty", "gateway", "llm", "prometheus" ], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Frosty Gateway Observability", "uid": "frosty-gateway-observability", "version": 2, "weekStart": "" }