Files
rippled/docker/telemetry/grafana/dashboards/job-queue.json
Pratik Mankawde cc4104b565 fix(telemetry): conditional filter-dimension legends + 30m span-nulls
Panel legends only showed the node (service_instance_id); the perf-iac
filter dimensions (xrpl_branch, xrpl_node_role) never appeared, so
baseline-vs-test series were indistinguishable.

Each node-referencing target now injects its curated name via
label_replace(...,"series",...) and uses the Prometheus "__auto" legend,
which shows only labels that vary across the displayed series. A
renameByRegex chain reshapes the auto output into
"Name [node, role]-[branch, work_item]", collapsing empty groups so a
dimension appears only when the filter widens it. Aggregating queries gain
xrpl_branch/xrpl_node_role in their by() so those labels survive; multi-
dimension legends keep their rich label set and append branch/role.

Quantile panels surfaced the quantile via the panel title (q$quantile),
since {{quantile}} is a dashboard variable, not a series label.

Timeseries panels now connect only null gaps shorter than 30m
(spanNulls=1800000): brief scrape gaps bridge, genuine outages stay broken.

Non-node panels (heatmaps, rows, tables, state timelines, category/bucket
series) are left unchanged. Applied to all 14 dashboards.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-14 11:30:00 +01:00

1291 lines
42 KiB
JSON

{
"annotations": {
"list": []
},
"description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"title": "Current Job Latency (p99 Gauge)",
"description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`",
"type": "gauge",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"orientation": "auto",
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")",
"legendFormat": "__auto"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "\u00b5s",
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 100000
},
{
"color": "red",
"value": 1000000
}
]
}
},
"overrides": []
},
"id": 1,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
},
{
"title": "Job Throughput Rate (Per Second)",
"description": "###### What this is:\n*Rate of jobs queued, started, and finished across all job types.*\n\n###### How it's computed:\n*Per-second rate of each cumulative job counter over a 5-minute window.*\n\n###### Reading it:\n*Queued, started, and finished tracking together means the queue keeps up.*\n\n###### Healthy range:\n*Workload-dependent; the three rates should stay roughly equal.*\n\n###### Watch for:\n*Queued rate persistently above finished rate, which indicates a growing backlog.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued / recordJobStarted / recordJobFinished`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Queued/s\", \"\", \"\")",
"legendFormat": "__auto"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\")",
"legendFormat": "__auto"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 10,
"axisLabel": "Operations / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 2,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
},
{
"title": "Per-Job-Type Queued Rate",
"description": "###### What this is:\n*Rate of jobs entering the queue, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the queued-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Identifies which job types generate the most queue activity.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single job type surging unexpectedly, which can point to a flood of a particular request or peer message.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": ["mean", "max"]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(topk(10, rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"$1\", \"job_type\", \"(.*)\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 5,
"axisLabel": "Operations / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 3,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le|job_type)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
},
{
"title": "Per-Job-Type Finish Rate",
"description": "###### What this is:\n*Rate of jobs completing, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the finished-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Compare against the queued rate per type to spot which types are falling behind.*\n\n###### Healthy range:\n*Workload-dependent; should match the per-type queued rate.*\n\n###### Watch for:\n*A type whose finish rate lags its queued rate, revealing where the backlog concentrates.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": ["mean", "max"]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(topk(10, rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[$__rate_interval])), \"series\", \"$1\", \"job_type\", \"(.*)\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 5,
"axisLabel": "Operations / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 4,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le|job_type)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
},
{
"title": "Job Queue Wait Time",
"description": "###### What this is:\n*Distribution of how long jobs sit in the queue before a worker picks them up (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; a widening gap between p75 and p99 signals occasional stalls.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait, an early sign of worker-thread saturation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p75 Wait\", \"\", \"\")",
"legendFormat": "__auto"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "\u00b5s",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 5,
"axisLabel": "Duration (\u03bcs)",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 5,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
},
{
"title": "Job Execution Time",
"description": "###### What this is:\n*Distribution of how long jobs run once started (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p75 Exec\", \"\", \"\")",
"legendFormat": "__auto"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "\u00b5s",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 5,
"axisLabel": "Duration (\u03bcs)",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 6,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
},
{
"title": "Per-Job-Type Execution Time (p99)",
"description": "###### What this is:\n*The 10 slowest job types ranked by p99 execution time.*\n\n###### How it's computed:\n*p99 derived from the run-time histogram per job_type over a 5-minute window, top 10 selected.*\n\n###### Reading it:\n*Highlights which job types cost the most CPU time at the tail.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A job type whose p99 grows over time, indicating a slow or degrading operation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 32
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": ["mean", "max"]
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\"}[5m])))), \"series\", \"$1\", \"job_type\", \"(.*)\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "\u00b5s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 5,
"axisLabel": "Duration (\u03bcs)",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 7,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le|job_type)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
},
{
"title": "Transaction Overflow Rate",
"description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate \u2014 the node is dropping transaction jobs because the queue is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 40
},
"options": {
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Overflows/min\", \"\", \"\")",
"legendFormat": "__auto"
}
],
"fieldConfig": {
"defaults": {
"unit": "cpm",
"thresholds": {
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 10
}
]
},
"custom": {
"axisLabel": "Overflows / Min",
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 10,
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 8,
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "/(?:__name__|job|metric|service_name|service_version|deployment_environment|instance|xrpl_network_type|le)=\"[^\"]*\",?\\s*/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\{(?:series=\"([^\"]*)\",?\\s*)?(?:service_instance_id=\"([^\"]*)\",?\\s*)?(?:xrpl_branch=\"([^\"]*)\",?\\s*)?(?:xrpl_node_role=\"([^\"]*)\",?\\s*)?(?:xrpl_work_item=\"([^\"]*)\",?\\s*)?\\}/g",
"renamePattern": "$1 [$2, $4]-[$3, $5]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/,\\s*\\]/g",
"renamePattern": "]"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[,\\s*/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]-\\[/g",
"renamePattern": "["
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/-\\[\\]/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\[\\]\\s*$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s*\\[\\]\\s*$/g",
"renamePattern": "all"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/\\s{2,}/g",
"renamePattern": " "
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^\\s+|\\s+$/g",
"renamePattern": ""
}
},
{
"id": "renameByRegex",
"options": {
"regex": "/^$/g",
"renamePattern": "all"
}
}
]
}
],
"schemaVersion": 39,
"tags": ["node", "jobqueue"],
"templating": {
"list": [
{
"name": "DS_PROMETHEUS",
"type": "datasource",
"label": "Prometheus",
"query": "prometheus",
"regex": "",
"current": {},
"hide": 0,
"refresh": 1,
"includeAll": false,
"multi": false,
"options": []
},
{
"name": "service_name",
"label": "Service Name",
"description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
"type": "query",
"query": "label_values(service_name)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "xrpld",
"value": "xrpld"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "deployment_environment",
"label": "Environment",
"description": "Filter by deployment tier [local / ci / test / prod]",
"type": "query",
"query": "label_values(deployment_environment)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_network_type",
"label": "Network",
"description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]",
"type": "query",
"query": "label_values(xrpl_network_type)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_work_item",
"label": "Work Item",
"description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)",
"type": "query",
"query": "label_values(xrpl_work_item)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_branch",
"label": "Branch",
"description": "Filter by comparison side (baseline:<ref>:<commit> / test:<ref>:<commit>)",
"type": "query",
"query": "label_values(xrpl_branch)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_node_role",
"label": "Node Role",
"description": "Filter by node role (validator / peer)",
"type": "query",
"query": "label_values(xrpl_node_role)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "node",
"label": "Node",
"description": "Filter by rippled node (service.instance.id)",
"type": "query",
"query": "label_values(target_info, service_instance_id)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "job_type",
"label": "Job Type",
"description": "Filter by job type",
"type": "query",
"query": "label_values(job_queued_total, job_type)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Job Queue Analysis",
"uid": "job-queue",
"version": 1,
"refresh": "10s"
}