1657 lines
43 KiB
JSON
Executable File
1657 lines
43 KiB
JSON
Executable File
{
|
|
"annotations": {
|
|
"list": []
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Span RED metrics (spanmetrics connector -> Prometheus)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "timeseries",
|
|
"title": "Span Rate by Operation (calls/s)",
|
|
"description": "Rate of every span the gateway emitted, from traces_span_metrics_calls_total. This is the request-rate half of RED and replaces browsing traces in the retired Jaeger UI.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (span_name) (rate(traces_span_metrics_calls_total[$__rate_interval]))",
|
|
"legendFormat": "{{span_name}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "Span Error Rate (%)",
|
|
"description": "Share of spans whose status_code is STATUS_CODE_ERROR, per operation. The errors half of RED.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (span_name) (rate(traces_span_metrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / clamp_min(sum by (span_name) (rate(traces_span_metrics_calls_total[$__rate_interval])), 1e-9)",
|
|
"legendFormat": "{{span_name}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "timeseries",
|
|
"title": "Span Duration percentiles (ms)",
|
|
"description": "p50/p95/p99 from the traces_span_metrics_duration_milliseconds histogram, per operation. The duration half of RED.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le, span_name) (rate(traces_span_metrics_duration_milliseconds_bucket[$__rate_interval])))",
|
|
"legendFormat": "p50 {{span_name}}"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le, span_name) (rate(traces_span_metrics_duration_milliseconds_bucket[$__rate_interval])))",
|
|
"legendFormat": "p95 {{span_name}}"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le, span_name) (rate(traces_span_metrics_duration_milliseconds_bucket[$__rate_interval])))",
|
|
"legendFormat": "p99 {{span_name}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "heatmap",
|
|
"title": "llm.call duration distribution (heatmap)",
|
|
"description": "Bucket distribution of upstream model-call latency, from the spanmetrics histogram filtered to the llm.call span.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 9
|
|
},
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 1,
|
|
"color": {
|
|
"mode": "scheme",
|
|
"scheme": "Turbo",
|
|
"steps": 64,
|
|
"reverse": false,
|
|
"exponent": 0.5,
|
|
"fill": "dark-orange"
|
|
},
|
|
"yAxis": {
|
|
"unit": "ms",
|
|
"axisPlacement": "left"
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"cellValues": {},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 1e-09
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (le) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=\"llm.call\"}[$__rate_interval]))",
|
|
"format": "heatmap",
|
|
"legendFormat": "{{le}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "row",
|
|
"title": "LLM calls by gen_ai.* span attributes",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 17
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "llm.call Rate by Provider and Model (calls/s)",
|
|
"description": "The gen_ai.provider.name and gen_ai.request.model span attributes are promoted to labels by the connector's `dimensions` config, so upstream call volume is attributable per model.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 18
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (gen_ai_provider_name, gen_ai_request_model) (rate(traces_span_metrics_calls_total{span_name=\"llm.call\"}[$__rate_interval]))",
|
|
"legendFormat": "{{gen_ai_provider_name}} / {{gen_ai_request_model}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "timeseries",
|
|
"title": "llm.call p95 Latency by Model (ms)",
|
|
"description": "Upstream latency broken down by model - which model is slow, not just that something is.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 18
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=\"llm.call\"}[$__rate_interval])))",
|
|
"legendFormat": "{{gen_ai_request_model}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "timeseries",
|
|
"title": "Finish Reasons (calls/s)",
|
|
"description": "gen_ai.response.finish_reason per second. A rising `length` share means responses are being truncated by max_tokens.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (gen_ai_response_finish_reason) (rate(traces_span_metrics_calls_total{span_name=\"llm.call\"}[$__rate_interval]))",
|
|
"legendFormat": "{{gen_ai_response_finish_reason}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "timeseries",
|
|
"title": "Streaming vs Non-Streaming (calls/s)",
|
|
"description": "gen_ai.stream split of upstream calls.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 26
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (gen_ai_stream) (rate(traces_span_metrics_calls_total{span_name=\"llm.call\"}[$__rate_interval]))",
|
|
"legendFormat": "stream={{gen_ai_stream}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "row",
|
|
"title": "Collector health (otelcol_* self-telemetry)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 34
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "stat",
|
|
"title": "Spans Accepted (total)",
|
|
"description": "otelcol_receiver_accepted_spans - spans the OTLP receiver took in. Zero here means the gateway is not exporting (check OTEL_EXPORTER_OTLP_ENDPOINT).",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 35
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(otelcol_receiver_accepted_spans)",
|
|
"legendFormat": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "stat",
|
|
"title": "Spans Refused (total)",
|
|
"description": "otelcol_receiver_refused_spans - spans rejected at the receiver. Anything above zero is data loss.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 35
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(otelcol_receiver_refused_spans)",
|
|
"legendFormat": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 14,
|
|
"type": "stat",
|
|
"title": "Export Failures (total)",
|
|
"description": "otelcol_exporter_send_failed_spans + _metric_points. Above zero means the collector could not hand data on.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 35
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(otelcol_exporter_send_failed_spans) + sum(otelcol_exporter_send_failed_metric_points)",
|
|
"legendFormat": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 15,
|
|
"type": "stat",
|
|
"title": "Collector Up",
|
|
"description": "Prometheus `up` for the otel-collector scrape job.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 35
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "up{job=\"otel-collector\"}",
|
|
"legendFormat": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 16,
|
|
"type": "timeseries",
|
|
"title": "Receiver Throughput (spans/s)",
|
|
"description": "Accepted vs refused span rate at the OTLP receiver, by transport.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "cps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (transport) (rate(otelcol_receiver_accepted_spans[$__rate_interval]))",
|
|
"legendFormat": "accepted {{transport}}"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (transport) (rate(otelcol_receiver_refused_spans[$__rate_interval]))",
|
|
"legendFormat": "refused {{transport}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 17,
|
|
"type": "timeseries",
|
|
"title": "Exporter Throughput (items/s)",
|
|
"description": "Spans and metric points the collector successfully sent, and failures, per exporter.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 41
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "cps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))",
|
|
"legendFormat": "sent spans {{exporter}}"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (exporter) (rate(otelcol_exporter_sent_metric_points[$__rate_interval]))",
|
|
"legendFormat": "sent metric points {{exporter}}"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))",
|
|
"legendFormat": "FAILED spans {{exporter}}"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval]))",
|
|
"legendFormat": "FAILED metric points {{exporter}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 18,
|
|
"type": "timeseries",
|
|
"title": "Batch Processor Send Size",
|
|
"description": "otelcol_processor_batch_batch_send_size percentiles - how full each outgoing batch is. Consistently tiny batches with a high timeout-trigger rate means traffic is below the batch threshold.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 49
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(otelcol_processor_batch_batch_send_size_bucket[$__rate_interval])))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(otelcol_processor_batch_batch_send_size_bucket[$__rate_interval])))",
|
|
"legendFormat": "p95"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 19,
|
|
"type": "timeseries",
|
|
"title": "Batch Timeout Triggers & Cardinality",
|
|
"description": "otelcol_processor_batch_timeout_trigger_send (batches flushed by timeout rather than size) and metadata cardinality.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 49
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(rate(otelcol_processor_batch_timeout_trigger_send[$__rate_interval]))",
|
|
"legendFormat": "timeout-triggered sends/s"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(otelcol_processor_batch_metadata_cardinality)",
|
|
"legendFormat": "metadata cardinality"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 20,
|
|
"type": "row",
|
|
"title": "Scrape health",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 57
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 21,
|
|
"type": "timeseries",
|
|
"title": "Target Up",
|
|
"description": "Prometheus `up` for all three jobs: frosty-gateway, otel-collector, otel-spanmetrics. A flatline at 0 pinpoints which leg of the pipeline broke.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 58
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "up{job=~\"frosty-gateway|otel-collector|otel-spanmetrics\"}",
|
|
"legendFormat": "{{job}} ({{instance}})"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 22,
|
|
"type": "timeseries",
|
|
"title": "Scrape Duration (s)",
|
|
"description": "How long each scrape takes. A climbing gateway scrape duration usually means metric cardinality growth.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 58
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "scrape_duration_seconds{job=~\"frosty-gateway|otel-collector|otel-spanmetrics\"}",
|
|
"legendFormat": "{{job}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 23,
|
|
"type": "row",
|
|
"title": "Label cardinality guard",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 66
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 24,
|
|
"type": "stat",
|
|
"title": "Models Dropped From Labels",
|
|
"description": "CARDINALITY GUARD. Every spanmetrics `dimension` multiplies the Prometheus series count, and model ids are the one unbounded dimension. The gateway admits the first FROSTY_OTEL_MODEL_CARDINALITY_CAP distinct models (default 11) as `frosty.metrics.model` and folds every further one to \"other\".\n\nA non-zero value here means label detail is being dropped: metrics for the folded models are merged into the \"other\" series. NOTHING IS LOST FROM TRACES - the span in Tempo still carries the real gen_ai.request.model, so drill-down is unaffected.\n\nTo restore per-model breakdown, raise FROSTY_OTEL_MODEL_CARDINALITY_CAP and accept the extra series, or leave it and use Tempo for the long tail.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 67
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"noValue": "0"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(frosty_counter{name=\"telemetry.span_model_label_collapsed\"})",
|
|
"legendFormat": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 25,
|
|
"type": "stat",
|
|
"title": "Distinct Model Labels",
|
|
"description": "How many distinct `frosty.metrics.model` label values currently exist in the spanmetrics series, including the \"other\" bucket and \"unknown\". This is the number the cap exists to bound - it should never exceed the cap by more than those two synthetic values.\n\nCARDINALITY GUARD. Every spanmetrics `dimension` multiplies the Prometheus series count, and model ids are the one unbounded dimension. The gateway admits the first FROSTY_OTEL_MODEL_CARDINALITY_CAP distinct models (default 11) as `frosty.metrics.model` and folds every further one to \"other\".\n\nA non-zero value here means label detail is being dropped: metrics for the folded models are merged into the \"other\" series. NOTHING IS LOST FROM TRACES - the span in Tempo still carries the real gen_ai.request.model, so drill-down is unaffected.\n\nTo restore per-model breakdown, raise FROSTY_OTEL_MODEL_CARDINALITY_CAP and accept the extra series, or leave it and use Tempo for the long tail.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 67
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 14
|
|
}
|
|
]
|
|
},
|
|
"noValue": "0"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "count(count by (frosty_metrics_model) (traces_span_metrics_calls_total))",
|
|
"legendFormat": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 26,
|
|
"type": "timeseries",
|
|
"title": "Label Collapse Rate (events/s)",
|
|
"description": "Rate at which spans are having their model label folded to \"other\". A flat zero means every model fits inside the cap. A sustained non-zero rate means you are routing to more models than the cap allows.\n\nCARDINALITY GUARD. Every spanmetrics `dimension` multiplies the Prometheus series count, and model ids are the one unbounded dimension. The gateway admits the first FROSTY_OTEL_MODEL_CARDINALITY_CAP distinct models (default 11) as `frosty.metrics.model` and folds every further one to \"other\".\n\nA non-zero value here means label detail is being dropped: metrics for the folded models are merged into the \"other\" series. NOTHING IS LOST FROM TRACES - the span in Tempo still carries the real gen_ai.request.model, so drill-down is unaffected.\n\nTo restore per-model breakdown, raise FROSTY_OTEL_MODEL_CARDINALITY_CAP and accept the extra series, or leave it and use Tempo for the long tail.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 67
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "cps",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "rate(frosty_counter{name=\"telemetry.span_model_label_collapsed\"}[$__rate_interval])",
|
|
"legendFormat": "collapsed/s"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 27,
|
|
"type": "timeseries",
|
|
"title": "Series Count by Dimension",
|
|
"description": "Distinct label values per spanmetrics dimension. Each of these multiplies the total series count, so this is the panel to watch before adding a new `dimension` to the collector's spanmetrics config.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 73
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "none",
|
|
"group": "A"
|
|
},
|
|
"axisPlacement": "auto"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "count(count by (frosty_metrics_model) (traces_span_metrics_calls_total))",
|
|
"legendFormat": "model (bounded)"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "count(count by (gen_ai_provider_name) (traces_span_metrics_calls_total))",
|
|
"legendFormat": "provider"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "count(count by (gen_ai_response_finish_reason) (traces_span_metrics_calls_total))",
|
|
"legendFormat": "finish_reason"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "count(count by (span_name) (traces_span_metrics_calls_total))",
|
|
"legendFormat": "span_name"
|
|
},
|
|
{
|
|
"refId": "E",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "count(traces_span_metrics_calls_total)",
|
|
"legendFormat": "TOTAL series"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 28,
|
|
"type": "row",
|
|
"title": "Traces (Tempo)",
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 80
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 29,
|
|
"type": "table",
|
|
"title": "Recent llm.call traces",
|
|
"description": "Live TraceQL search against Tempo: {name=\"llm.call\"}. Click a Trace ID to open the full span tree - this is the per-request drill-down that the spanmetrics panels above cannot give you. The span carries the REAL gen_ai.request.model even when the metric label has been folded to \"other\".",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 81
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "{name=\"llm.call\"}",
|
|
"limit": 20
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 39,
|
|
"tags": [
|
|
"frosty",
|
|
"opentelemetry",
|
|
"collector",
|
|
"prometheus"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "Prometheus",
|
|
"value": "prometheus"
|
|
},
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Datasource",
|
|
"multi": false,
|
|
"name": "DS_PROMETHEUS",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"queryValue": "",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {},
|
|
"timezone": "",
|
|
"title": "OpenTelemetry Collector",
|
|
"uid": "frosty-otel-collector",
|
|
"version": 3,
|
|
"weekStart": ""
|
|
}
|