ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-08-22 23:00:55 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
df01ba5d3d2bbd6fc6ef5527cb04f1e24d1bb0bb
rippled/docker/telemetry/grafana/dashboards/job-queue.json
Pratik Mankawde 556d620cbe fix(telemetry): make job-queue legends resolve every label they name 
Post-deploy verification against Grafana Cloud found two problems on the
job-queue dashboard.

Every aggregating panel's displayName names four labels, but the queries
aggregate with `sum by (...)` lists that omit `xrpl_work_item`, so that
part of the legend rendered as an unresolved reference. Pre-existing, but
splitting the wait and execution panels by job type made the legend
load-bearing, so it is fixed here: `xrpl_work_item` is added to the ten
affected sum-by lists.

The two panels split by job type in the previous commit also kept their
seven-section descriptions. They now carry the same ten sections as the
rest, and their glossary links point at the real `job-queue-job-type`
anchor rather than a `job-queue` anchor that does not exist.

Verified no dashboard links a glossary anchor that is absent.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-25 13:08:18 +01:00

655 lines
30 KiB
JSON
Raw Blame History

{
"annotations": {
"list": []
},
"description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"title": "Current Job Latency (p99 Gauge)",
"description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`",
"type": "gauge",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"options": {
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"orientation": "auto",
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "µs",
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 100000
},
{
"color": "red",
"value": 1000000
}
]
}
},
"overrides": []
},
"id": 1
},
{
"title": "Job Throughput Rate (Per Second)",
"description": "###### What this is:\n*Rate of jobs queued, started, and finished across all job types.*\n\n###### How it's computed:\n*Per-second rate of each cumulative job counter over a 5-minute window.*\n\n###### Reading it:\n*Queued, started, and finished tracking together means the queue keeps up.*\n\n###### Healthy range:\n*Workload-dependent; the three rates should stay roughly equal.*\n\n###### Watch for:\n*Queued rate persistently above finished rate, which indicates a growing backlog.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued / recordJobStarted / recordJobFinished`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"options": {
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Queued/s\", \"\", \"\")"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\")"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 10,
"axisLabel": "Operations / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 2
},
{
"title": "Per-Job-Type Queued Rate",
"description": "###### What this is:\n*Rate of jobs entering the queue, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the queued-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Identifies which job types generate the most queue activity.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single job type surging unexpectedly, which can point to a flood of a particular request or peer message.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobQueued`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"options": {
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": ["mean", "max"]
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 5,
"axisLabel": "Operations / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 3
},
{
"title": "Per-Job-Type Finish Rate",
"description": "###### What this is:\n*Rate of jobs completing, broken down by job type (top 10).*\n\n###### How it's computed:\n*Per-second rate of the finished-job counter per job_type over a 5-minute window.*\n\n###### Reading it:\n*Compare against the queued rate per type to spot which types are falling behind.*\n\n###### Healthy range:\n*Workload-dependent; should match the per-type queued rate.*\n\n###### Watch for:\n*A type whose finish rate lags its queued rate, revealing where the backlog concentrates.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 24
},
"options": {
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": ["mean", "max"]
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(topk(10, sum by (job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[$__rate_interval]))), \"series\", \"$1\", \"job_type\", \"(.*)\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 5,
"axisLabel": "Operations / Sec",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 4
},
{
"title": "Job Queue Wait Time",
"description": "###### What this is:\n*How long jobs sit in the queue before a worker picks them up, split by job type (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job wait-time histogram over a 5-minute window, kept per job type. Limited to the ten types with the highest wait so the legend stays readable.*\n\n###### Reading it:\n*Lower is better. The legend names the job type, so a single slow queue is identifiable rather than hidden in an all-types average. A widening p75-to-p99 gap on one type signals occasional stalls there.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond waits on an unloaded node.*\n\n###### Watch for:\n*Rising p99 wait on a capped type -- ledgerRequest, ledgerData and makeFetchPack have small concurrency limits, so they queue first. Cross-check the deferred gauge for that type.*\n\n###### Keywords:\n- **Job queue** *(per node)* — the worker pool that runs xrpld's background work; each unit of work is a job with a type.\n- **Dequeue wait** *(per node)* — time a job sits enqueued before a worker starts it, as distinct from how long it then runs.\n- **Concurrency limit** *(per node)* — the maximum number of jobs of one type allowed to run at once; work beyond it is deferred, not rejected.\n\n###### Computation boundary:\n*Result: Per node and job type — each series is one server's own value for one job type.*\n*Recorded in code as an OTel SDK histogram, then aggregated to percentiles by the Grafana query.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted`\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type) · [Concurrency limit](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#concurrency-limit)\n",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 32
},
"options": {
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(topk(10, histogram_quantile(0.75, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p75 Wait $1\", \"job_type\", \"(.*)\")"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p99 Wait $1\", \"job_type\", \"(.*)\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "µs",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 5,
"axisLabel": "Duration (μs)",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 5
},
{
"title": "Job Execution Time",
"description": "###### What this is:\n*How long jobs run once started, split by job type (p75 typical, p99 tail).*\n\n###### How it's computed:\n*Percentiles derived from the job run-time histogram over a 5-minute window, kept per job type. Limited to the ten slowest types so the legend stays readable.*\n\n###### Reading it:\n*Lower is better. The legend names the job type, so an expensive job type is identifiable rather than averaged away. Stable p75 with a controlled p99 is healthy.*\n\n###### Healthy range:\n*Workload-dependent, but stable over time.*\n\n###### Watch for:\n*Growing execution times, which point to CPU pressure or expensive individual jobs.*\n\n###### Keywords:\n- **Job queue** *(per node)* — the worker pool that runs xrpld's background work; each unit of work is a job with a type.\n- **Execution time** *(per node)* — time a job spends running after a worker picks it up, excluding its queue wait.\n\n###### Computation boundary:\n*Result: Per node and job type — each series is one server's own value for one job type.*\n*Recorded in code as an OTel SDK histogram, then aggregated to percentiles by the Grafana query.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)\n",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 32
},
"options": {
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(topk(10, histogram_quantile(0.75, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p75 Exec $1\", \"job_type\", \"(.*)\")"
},
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"p99 Exec $1\", \"job_type\", \"(.*)\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "µs",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 5,
"axisLabel": "Duration (μs)",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 6
},
{
"title": "Per-Job-Type Execution Time (p99)",
"description": "###### What this is:\n*The 10 slowest job types ranked by p99 execution time.*\n\n###### How it's computed:\n*p99 derived from the run-time histogram per job_type over a 5-minute window, top 10 selected.*\n\n###### Reading it:\n*Highlights which job types cost the most CPU time at the tail.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A job type whose p99 grows over time, indicating a slow or degrading operation.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobFinished`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 40
},
"options": {
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": ["mean", "max"]
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(topk(10, histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m])))), \"series\", \"$1\", \"job_type\", \"(.*)\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "µs",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 5,
"axisLabel": "Duration (μs)",
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 7
},
{
"title": "Transaction Overflow Rate",
"description": "###### What this is:\n*Rate at which transaction jobs are shed when the queue's transaction limit is exceeded.*\n\n###### How it's computed:\n*Per-second rate of the overflow counter over the dashboard's rate interval, scaled to per minute. The counter is observed from the overlay's cumulative overflow tally.*\n\n###### Reading it:\n*Near zero is healthy; a rising rate means the job queue is shedding transaction work under load.*\n\n###### Healthy range:\n*0 overflows per minute.*\n\n###### Watch for:\n*Any sustained non-zero rate — the node is dropping transaction jobs because the queue is saturated.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerParityCounters (observed from Overlay::getJqTransOverflow)`",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 48
},
"options": {
"tooltip": {
"maxHeight": 600,
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"expr": "label_replace(rate(jq_trans_overflow_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) * 60, \"series\", \"Overflows/min\", \"\", \"\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
"unit": "cpm",
"thresholds": {
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 10
}
]
},
"custom": {
"axisLabel": "Overflows / Min",
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 10,
"spanNulls": 1800000,
"insertNulls": false,
"showPoints": "auto",
"pointSize": 3
},
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"id": 8
}
],
"schemaVersion": 39,
"tags": ["node", "jobqueue"],
"templating": {
"list": [
{
"name": "service_name",
"label": "Service Name",
"description": "Filter by service.name (e.g. xrpld, xrpld-validator)",
"type": "query",
"query": "label_values(service_name)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "xrpld",
"value": "xrpld"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "deployment_environment",
"label": "Environment",
"description": "Filter by deployment tier [local / ci / test / prod]",
"type": "query",
"query": "label_values(deployment_environment)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_network_type",
"label": "Network",
"description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]",
"type": "query",
"query": "label_values(xrpl_network_type)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_work_item",
"label": "Work Item",
"description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)",
"type": "query",
"query": "label_values(xrpl_work_item)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_branch",
"label": "Branch",
"description": "Filter by comparison side (baseline:<ref>:<commit> / test:<ref>:<commit>)",
"type": "query",
"query": "label_values(xrpl_branch)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "xrpl_node_role",
"label": "Node Role",
"description": "Filter by node role (validator / peer)",
"type": "query",
"query": "label_values(xrpl_node_role)",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "node",
"label": "Node",
"description": "Filter by rippled node (service.instance.id)",
"type": "query",
"query": "label_values(target_info, service_instance_id)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "job_type",
"label": "Job Type",
"description": "Filter by job type",
"type": "query",
"query": "label_values(job_queued_total, job_type)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
},
{
"name": "handler",
"label": "Handler",
"description": "Filter by enqueueing handler (sanitised addJob name, e.g. RcvGetLedger)",
"type": "query",
"query": "label_values(job_queued_total, handler)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"includeAll": true,
"allValue": ".*",
"current": {
"text": "All",
"value": "$__all"
},
"multi": true,
"refresh": 2,
"sort": 1
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Job Queue Analysis",
"uid": "job-queue",
"version": 1,
"refresh": "5s"
}
Reference in New Issue View Git Blame Copy Permalink