{ "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": false, "panels": [ { "id": 1, "type": "row", "title": "Span RED metrics (spanmetrics connector -> Prometheus)", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 2, "type": "timeseries", "title": "Span Rate by Operation (calls/s)", "description": "Rate of every span the gateway emitted, from traces_span_metrics_calls_total. This is the request-rate half of RED and replaces browsing traces in the retired Jaeger UI.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (span_name) (rate(traces_span_metrics_calls_total[$__rate_interval]))", "legendFormat": "{{span_name}}" } ] }, { "id": 3, "type": "timeseries", "title": "Span Error Rate (%)", "description": "Share of spans whose status_code is STATUS_CODE_ERROR, per operation. The errors half of RED.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, "fieldConfig": { "defaults": { "unit": "percentunit", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (span_name) (rate(traces_span_metrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / clamp_min(sum by (span_name) (rate(traces_span_metrics_calls_total[$__rate_interval])), 1e-9)", "legendFormat": "{{span_name}}" } ] }, { "id": 4, "type": "timeseries", "title": "Span Duration percentiles (ms)", "description": "p50/p95/p99 from the traces_span_metrics_duration_milliseconds histogram, per operation. The duration half of RED.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "fieldConfig": { "defaults": { "unit": "ms", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le, span_name) (rate(traces_span_metrics_duration_milliseconds_bucket[$__rate_interval])))", "legendFormat": "p50 {{span_name}}" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le, span_name) (rate(traces_span_metrics_duration_milliseconds_bucket[$__rate_interval])))", "legendFormat": "p95 {{span_name}}" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le, span_name) (rate(traces_span_metrics_duration_milliseconds_bucket[$__rate_interval])))", "legendFormat": "p99 {{span_name}}" } ] }, { "id": 5, "type": "heatmap", "title": "llm.call duration distribution (heatmap)", "description": "Bucket distribution of upstream model-call latency, from the spanmetrics histogram filtered to the llm.call span.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "options": { "calculate": false, "cellGap": 1, "color": { "mode": "scheme", "scheme": "Turbo", "steps": 64, "reverse": false, "exponent": 0.5, "fill": "dark-orange" }, "yAxis": { "unit": "ms", "axisPlacement": "left" }, "legend": { "show": true }, "tooltip": { "show": true, "yHistogram": true }, "rowsFrame": { "layout": "auto" }, "cellValues": {}, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (le) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=\"llm.call\"}[$__rate_interval]))", "format": "heatmap", "legendFormat": "{{le}}" } ] }, { "id": 6, "type": "row", "title": "LLM calls by gen_ai.* span attributes", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "panels": [] }, { "id": 7, "type": "timeseries", "title": "llm.call Rate by Provider and Model (calls/s)", "description": "The gen_ai.provider.name and gen_ai.request.model span attributes are promoted to labels by the connector's `dimensions` config, so upstream call volume is attributable per model.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (gen_ai_provider_name, gen_ai_request_model) (rate(traces_span_metrics_calls_total{span_name=\"llm.call\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_provider_name}} / {{gen_ai_request_model}}" } ] }, { "id": 8, "type": "timeseries", "title": "llm.call p95 Latency by Model (ms)", "description": "Upstream latency broken down by model - which model is slow, not just that something is.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, "fieldConfig": { "defaults": { "unit": "ms", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=\"llm.call\"}[$__rate_interval])))", "legendFormat": "{{gen_ai_request_model}}" } ] }, { "id": 9, "type": "timeseries", "title": "Finish Reasons (calls/s)", "description": "gen_ai.response.finish_reason per second. A rising `length` share means responses are being truncated by max_tokens.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (gen_ai_response_finish_reason) (rate(traces_span_metrics_calls_total{span_name=\"llm.call\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_response_finish_reason}}" } ] }, { "id": 10, "type": "timeseries", "title": "Streaming vs Non-Streaming (calls/s)", "description": "gen_ai.stream split of upstream calls.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (gen_ai_stream) (rate(traces_span_metrics_calls_total{span_name=\"llm.call\"}[$__rate_interval]))", "legendFormat": "stream={{gen_ai_stream}}" } ] }, { "id": 11, "type": "row", "title": "Collector health (otelcol_* self-telemetry)", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, "panels": [] }, { "id": 12, "type": "stat", "title": "Spans Accepted (total)", "description": "otelcol_receiver_accepted_spans - spans the OTLP receiver took in. Zero here means the gateway is not exporting (check OTEL_EXPORTER_OTLP_ENDPOINT).", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 35 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(otelcol_receiver_accepted_spans)", "legendFormat": "" } ] }, { "id": 13, "type": "stat", "title": "Spans Refused (total)", "description": "otelcol_receiver_refused_spans - spans rejected at the receiver. Anything above zero is data loss.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 35 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(otelcol_receiver_refused_spans)", "legendFormat": "" } ] }, { "id": 14, "type": "stat", "title": "Export Failures (total)", "description": "otelcol_exporter_send_failed_spans + _metric_points. Above zero means the collector could not hand data on.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 35 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(otelcol_exporter_send_failed_spans) + sum(otelcol_exporter_send_failed_metric_points)", "legendFormat": "" } ] }, { "id": 15, "type": "stat", "title": "Collector Up", "description": "Prometheus `up` for the otel-collector scrape job.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 35 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto", "colorMode": "value", "graphMode": "area", "justifyMode": "auto" }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "up{job=\"otel-collector\"}", "legendFormat": "" } ] }, { "id": 16, "type": "timeseries", "title": "Receiver Throughput (spans/s)", "description": "Accepted vs refused span rate at the OTLP receiver, by transport.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "fieldConfig": { "defaults": { "unit": "cps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (transport) (rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "legendFormat": "accepted {{transport}}" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (transport) (rate(otelcol_receiver_refused_spans[$__rate_interval]))", "legendFormat": "refused {{transport}}" } ] }, { "id": 17, "type": "timeseries", "title": "Exporter Throughput (items/s)", "description": "Spans and metric points the collector successfully sent, and failures, per exporter.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "fieldConfig": { "defaults": { "unit": "cps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "legendFormat": "sent spans {{exporter}}" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (exporter) (rate(otelcol_exporter_sent_metric_points[$__rate_interval]))", "legendFormat": "sent metric points {{exporter}}" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "legendFormat": "FAILED spans {{exporter}}" }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval]))", "legendFormat": "FAILED metric points {{exporter}}" } ] }, { "id": 18, "type": "timeseries", "title": "Batch Processor Send Size", "description": "otelcol_processor_batch_batch_send_size percentiles - how full each outgoing batch is. Consistently tiny batches with a high timeout-trigger rate means traffic is below the batch threshold.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(otelcol_processor_batch_batch_send_size_bucket[$__rate_interval])))", "legendFormat": "p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(otelcol_processor_batch_batch_send_size_bucket[$__rate_interval])))", "legendFormat": "p95" } ] }, { "id": 19, "type": "timeseries", "title": "Batch Timeout Triggers & Cardinality", "description": "otelcol_processor_batch_timeout_trigger_send (batches flushed by timeout rather than size) and metadata cardinality.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(otelcol_processor_batch_timeout_trigger_send[$__rate_interval]))", "legendFormat": "timeout-triggered sends/s" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(otelcol_processor_batch_metadata_cardinality)", "legendFormat": "metadata cardinality" } ] }, { "id": 20, "type": "row", "title": "Scrape health", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 57 }, "panels": [] }, { "id": 21, "type": "timeseries", "title": "Target Up", "description": "Prometheus `up` for all three jobs: frosty-gateway, otel-collector, otel-spanmetrics. A flatline at 0 pinpoints which leg of the pipeline broke.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 58 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "up{job=~\"frosty-gateway|otel-collector|otel-spanmetrics\"}", "legendFormat": "{{job}} ({{instance}})" } ] }, { "id": 22, "type": "timeseries", "title": "Scrape Duration (s)", "description": "How long each scrape takes. A climbing gateway scrape duration usually means metric cardinality growth.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 58 }, "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "scrape_duration_seconds{job=~\"frosty-gateway|otel-collector|otel-spanmetrics\"}", "legendFormat": "{{job}}" } ] }, { "id": 23, "type": "row", "title": "Label cardinality guard", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 66 }, "panels": [] }, { "id": 24, "type": "stat", "title": "Models Dropped From Labels", "description": "CARDINALITY GUARD. Every spanmetrics `dimension` multiplies the Prometheus series count, and model ids are the one unbounded dimension. The gateway admits the first FROSTY_OTEL_MODEL_CARDINALITY_CAP distinct models (default 11) as `frosty.metrics.model` and folds every further one to \"other\".\n\nA non-zero value here means label detail is being dropped: metrics for the folded models are merged into the \"other\" series. NOTHING IS LOST FROM TRACES - the span in Tempo still carries the real gen_ai.request.model, so drill-down is unaffected.\n\nTo restore per-model breakdown, raise FROSTY_OTEL_MODEL_CARDINALITY_CAP and accept the extra series, or leave it and use Tempo for the long tail.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 67 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] }, "noValue": "0" }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto", "colorMode": "background", "graphMode": "area", "justifyMode": "auto" }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(frosty_counter{name=\"telemetry.span_model_label_collapsed\"})", "legendFormat": "" } ] }, { "id": 25, "type": "stat", "title": "Distinct Model Labels", "description": "How many distinct `frosty.metrics.model` label values currently exist in the spanmetrics series, including the \"other\" bucket and \"unknown\". This is the number the cap exists to bound - it should never exceed the cap by more than those two synthetic values.\n\nCARDINALITY GUARD. Every spanmetrics `dimension` multiplies the Prometheus series count, and model ids are the one unbounded dimension. The gateway admits the first FROSTY_OTEL_MODEL_CARDINALITY_CAP distinct models (default 11) as `frosty.metrics.model` and folds every further one to \"other\".\n\nA non-zero value here means label detail is being dropped: metrics for the folded models are merged into the \"other\" series. NOTHING IS LOST FROM TRACES - the span in Tempo still carries the real gen_ai.request.model, so drill-down is unaffected.\n\nTo restore per-model breakdown, raise FROSTY_OTEL_MODEL_CARDINALITY_CAP and accept the extra series, or leave it and use Tempo for the long tail.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 67 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 14 } ] }, "noValue": "0" }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto", "colorMode": "background", "graphMode": "area", "justifyMode": "auto" }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(count by (frosty_metrics_model) (traces_span_metrics_calls_total))", "legendFormat": "" } ] }, { "id": 26, "type": "timeseries", "title": "Label Collapse Rate (events/s)", "description": "Rate at which spans are having their model label folded to \"other\". A flat zero means every model fits inside the cap. A sustained non-zero rate means you are routing to more models than the cap allows.\n\nCARDINALITY GUARD. Every spanmetrics `dimension` multiplies the Prometheus series count, and model ids are the one unbounded dimension. The gateway admits the first FROSTY_OTEL_MODEL_CARDINALITY_CAP distinct models (default 11) as `frosty.metrics.model` and folds every further one to \"other\".\n\nA non-zero value here means label detail is being dropped: metrics for the folded models are merged into the \"other\" series. NOTHING IS LOST FROM TRACES - the span in Tempo still carries the real gen_ai.request.model, so drill-down is unaffected.\n\nTo restore per-model breakdown, raise FROSTY_OTEL_MODEL_CARDINALITY_CAP and accept the extra series, or leave it and use Tempo for the long tail.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 67 }, "fieldConfig": { "defaults": { "unit": "cps", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(frosty_counter{name=\"telemetry.span_model_label_collapsed\"}[$__rate_interval])", "legendFormat": "collapsed/s" } ] }, { "id": 27, "type": "timeseries", "title": "Series Count by Dimension", "description": "Distinct label values per spanmetrics dimension. Each of these multiplies the total series count, so this is the panel to watch before adding a new `dimension` to the collector's spanmetrics config.", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 73 }, "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(count by (frosty_metrics_model) (traces_span_metrics_calls_total))", "legendFormat": "model (bounded)" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(count by (gen_ai_provider_name) (traces_span_metrics_calls_total))", "legendFormat": "provider" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(count by (gen_ai_response_finish_reason) (traces_span_metrics_calls_total))", "legendFormat": "finish_reason" }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(count by (span_name) (traces_span_metrics_calls_total))", "legendFormat": "span_name" }, { "refId": "E", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(traces_span_metrics_calls_total)", "legendFormat": "TOTAL series" } ] }, { "id": 28, "type": "row", "title": "Traces (Tempo)", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 80 }, "panels": [] }, { "id": 29, "type": "table", "title": "Recent llm.call traces", "description": "Live TraceQL search against Tempo: {name=\"llm.call\"}. Click a Trace ID to open the full span tree - this is the per-request drill-down that the spanmetrics panels above cannot give you. The span carries the REAL gen_ai.request.model even when the metric label has been folded to \"other\".", "datasource": { "type": "tempo", "uid": "tempo" }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 81 }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true } }, "overrides": [] }, "options": { "showHeader": true }, "targets": [ { "refId": "A", "datasource": { "type": "tempo", "uid": "tempo" }, "queryType": "traceql", "query": "{name=\"llm.call\"}", "limit": 20 } ] } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "frosty", "opentelemetry", "collector", "prometheus" ], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "OpenTelemetry Collector", "uid": "frosty-otel-collector", "version": 3, "weekStart": "" }