{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.4.1" }, { "type": "panel", "id": "piechart", "name": "Pie chart", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" }, { "type": "panel", "id": "text", "name": "Text", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "$$hashKey": "object:56", "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "limit": 100, "name": "Annotations & Alerts", "showIn": 0, "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Dashboard to monitor multiple Teku instances.", "editable": true, "fiscalYearStartMonth": 0, "gnetId": 16737, "graphTooltip": 1, "id": null, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "Teku Website", "tooltip": "", "type": "link", "url": "https://consensys.net/knowledge-base/ethereum-2/teku" }, { "asDropdown": false, "icon": "doc", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "Teku Documentation", "tooltip": "", "type": "link", "url": "https://docs.teku.consensys.net/" }, { "asDropdown": false, "icon": "external link", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "Teku GitHub", "tooltip": "", "type": "link", "url": "https://github.com/ConsenSys/teku" } ], "liveNow": false, "panels": [ { "collapsed": false, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 79, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Validator", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of local validators", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "exited_slashed" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "red", "value": 1 } ] } } ] }, { "matcher": { "id": "byName", "options": "active_slashed" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "red", "value": 1 } ] } } ] }, { "matcher": { "id": "byName", "options": "unknown" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "yellow", "value": 1 } ] } } ] } ] }, "gridPos": { "h": 4, "w": 20, "x": 0, "y": 1 }, "id": 36, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sort_desc(sum by (status) (validator_local_validator_counts{instance=~\"$system\"}))", "instant": true, "interval": "", "legendFormat": "{{status}}", "refId": "A" } ], "title": "Local Validators", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false, "minWidth": 0 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "version" }, "properties": [ { "id": "displayName", "value": "Version" } ] }, { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "displayName", "value": "Count" }, { "id": "custom.width", "value": 50 } ] } ] }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "id": 126, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": false }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum by (version) (beacon_teku_version{instance=~\"$system\"}) or sum by (version) (beacon_teku_version_total{instance=~\"$system\"})", "format": "table", "instant": true, "interval": "", "legendFormat": "{{version}}", "refId": "A" } ], "title": "Teku Versions", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of blocks published", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 0, "y": 5 }, "id": 37, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "value", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(validator_beacon_node_requests_total{instance=~\"$system\",method=\"publish_block\",outcome=\"success\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Published Blocks", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of attestations published", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 5, "y": 5 }, "id": 38, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "value", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(validator_beacon_node_requests_total{instance=~\"$system\",method=\"publish_attestation\",outcome=\"success\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Published Attestations", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of aggregates published", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 10, "y": 5 }, "id": 39, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "value", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(validator_beacon_node_requests_total{instance=~\"$system\",method=\"publish_aggregate_and_proofs\",outcome=\"success\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Published Aggregates", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of sync committee messages published.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 15, "y": 5 }, "id": 67, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "value", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(validator_beacon_node_requests_total{instance=~\"$system\",method=\"send_sync_committee_messages\",outcome=\"success\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Published Sync Committee Messages", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total number of active validators in the network.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "displayName": "", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 5 }, "id": 44, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "value", "wideLayout": true }, "pluginVersion": "10.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "max(beacon_current_active_validators{instance=~\"$system\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Total Validators", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of attestations in each epoch with the correct head root, correct target root and that were successfully included in blocks.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 8 }, "id": 41, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_max(avg((validator_performance_included_attestations{instance=~\"$system\"} / validator_performance_expected_attestations{instance=~\"$system\"}) >= 0), 1)", "interval": "", "legendFormat": "{{instance}} Included", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_max(avg((validator_performance_correct_head_block_count{instance=~\"$system\"} / validator_performance_included_attestations{instance=~\"$system\"}) >= 0), 1)", "interval": "", "legendFormat": "{{instance}} Correct Head", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "avg((validator_performance_correct_target_count{instance=~\"$system\"} / validator_performance_included_attestations{instance=~\"$system\"}) >= 0)", "interval": "", "legendFormat": "{{instance}} Correct Target", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_max(avg(\n((\nvalidator_performance_correct_head_block_count{instance=~\"$system\"} - (validator_performance_expected_attestations{instance=~\"$system\"} - validator_performance_correct_head_block_count{instance=~\"$system\"}) +\nvalidator_performance_correct_target_count{instance=~\"$system\"} - (validator_performance_expected_attestations{instance=~\"$system\"} - validator_performance_correct_target_count{instance=~\"$system\"}) +\nvalidator_performance_included_attestations{instance=~\"$system\"} - (validator_performance_expected_attestations{instance=~\"$system\"} - validator_performance_included_attestations{instance=~\"$system\"}) +\n(0.875 * validator_performance_included_attestations{instance=~\"$system\"} / validator_performance_inclusion_distance_average{instance=~\"$system\"})\n) / validator_performance_expected_attestations{instance=~\"$system\"} / 3.875)>=0), 1)", "interval": "", "legendFormat": "{{instance}} Attestation Rewards Earned", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_max(avg((validator_performance_included_sync_committee_messages{instance=~\"$system\"} / validator_performance_expected_sync_committee_messages{instance=~\"$system\"}) >= 0), 1)", "hide": false, "interval": "", "legendFormat": "{{instance}} Included Signatures", "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_max(avg((validator_performance_produced_sync_committee_messages{instance=~\"$system\"} / validator_performance_expected_sync_committee_messages{instance=~\"$system\"}) >= 0), 1)", "hide": false, "interval": "", "legendFormat": "{{instance}} Produced Signatures", "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_max(avg((validator_performance_correct_sync_committee_messages{instance=~\"$system\"} / validator_performance_produced_sync_committee_messages{instance=~\"$system\"}) >= 0), 1)", "hide": false, "interval": "", "legendFormat": "{{instance}} Correct Signatures", "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "clamp_max(avg((validator_performance_produced_attestations{instance=~\"$system\"} / validator_performance_expected_attestations{instance=~\"$system\"}) >= 0), 1)", "hide": false, "interval": "", "legendFormat": "{{instance}} Produced", "range": true, "refId": "H" } ], "title": "Attestation Performance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of slots between when an attestation is created to when it is included in a block. The minimum possible inclusion distance is 1.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 8 }, "id": 43, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "validator_performance_inclusion_distance_min{instance=~\"$system\"}", "interval": "", "legendFormat": "{{instance}} Minimum", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "validator_performance_inclusion_distance_average{instance=~\"$system\"}", "interval": "", "legendFormat": "{{instance}} Average", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "validator_performance_inclusion_distance_max{instance=~\"$system\"}", "interval": "", "legendFormat": "{{instance}} Maximum", "refId": "C" } ], "title": "Attestation Inclusion Distance (slots)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of blocks produced in each epoch that were successfully included in the canonical chain.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 8 }, "id": 42, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "validator_performance_included_blocks{instance=~\"$system\"} / validator_performance_expected_blocks{instance=~\"$system\"}", "interval": "", "legendFormat": "{{instance}} Included Blocks", "refId": "A" } ], "title": "Included Blocks", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of attestations in each epoch with the correct head root, correct target root and that were successfully included in blocks.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 18 }, "id": 142, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_max((validator_performance_included_attestations{instance=~\"$system\"} / validator_performance_expected_attestations{instance=~\"$system\"} >= 0), 1)", "interval": "", "legendFormat": "{{instance}} Included", "refId": "C" } ], "title": "Attestation Inclusion Detail", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of attestations in each epoch with the correct head root, correct target root and that were successfully included in blocks.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 18 }, "id": 143, "options": { "legend": { "calcs": [ "min", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "clamp_max((validator_performance_correct_head_block_count{instance=~\"$system\"} / validator_performance_included_attestations{instance=~\"$system\"} >= 0), 1)", "interval": "", "legendFormat": "{{instance}} Correct Head", "range": true, "refId": "C" } ], "title": "Correct Head Detail", "type": "timeseries" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "id": 87, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,1\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,4000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[4000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[8000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 14, "w": 12, "x": 0, "y": 40 }, "id": 89, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (rate(validator_attestation_publication_delay{instance=~\"$system\"}[5m]))", "hide": false, "interval": "", "legendFormat": "{{interval}}", "refId": "B" } ], "title": "Attestation Production Delay", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,1\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,4000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[4000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[12000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[8000,12000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 14, "w": 12, "x": 12, "y": 40 }, "id": 90, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (rate(validator_block_publication_delay{instance=~\"$system\"}[$__range]))", "hide": false, "interval": "", "legendFormat": "{{interval}}", "refId": "B" } ], "title": "Block Production Delay", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,1\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,4000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[4000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[8000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "[0,1)" }, "properties": [ { "id": "color", "value": { "fixedColor": "#37872D", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 54 }, "id": 92, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(validator_attestation_publication_delay{instance=~\"$system\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "title": "Attestation Delays", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,1\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,4000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[4000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[12000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[8000,12000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 54 }, "id": 93, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(validator_block_publication_delay{instance=~\"$system\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "title": "Block Delays", "type": "piechart" } ], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Validator Timings", "type": "row" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, "id": 81, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Beacon Chain", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current slot according to system time", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 0, "y": 30 }, "id": 45, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "max by (network) (beacon_slot{instance=~\"$system\"})", "instant": true, "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Current slot", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Slot of the latest imported block", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 5, "y": 30 }, "id": 48, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "max by (network) (beacon_head_slot{instance=~\"$system\"})", "instant": true, "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Head slot", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current justified epoch", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 10, "y": 30 }, "id": 49, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "max by (network) (beacon_current_justified_epoch{instance=~\"$system\"})", "instant": true, "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Justified Epoch", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current finalized epoch", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 15, "y": 30 }, "id": 51, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "max by (network) (beacon_finalized_epoch{instance=~\"$system\"})", "instant": true, "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Finalized Epoch", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total number of peers connected to the selected nodes.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "#EAB839", "value": 25 }, { "color": "green", "value": 60 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 30 }, "id": 64, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (network) (beacon_peer_count{instance=~\"$system\"})", "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Peer Count", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The current epoch according to system time", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 0, "y": 33 }, "id": 46, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "max by (network) (beacon_epoch{instance=~\"$system\"})", "instant": true, "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Current Epoch", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of slots the head block is behind.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "yellow", "value": 3 }, { "color": "red", "value": 5 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 5, "y": 33 }, "id": 47, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "uniqueValues" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "max without (alerting, job, instance, client) (beacon_slot{instance=~\"$system\"} - beacon_head_slot{instance=~\"$system\"})", "instant": true, "interval": "", "legendFormat": "__auto", "refId": "A" } ], "title": "Slots Behind", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of epochs that justification has been delayed.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "yellow", "value": 2 }, { "color": "red", "value": 5 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 10, "y": 33 }, "id": 50, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "clamp_min(min without (instance, job, alerting, client) (beacon_epoch{instance=~\"$system\"} - beacon_current_justified_epoch{instance=~\"$system\"} - 1), 0)", "instant": false, "interval": "", "legendFormat": "{{network}}", "range": true, "refId": "A" } ], "title": "Justification Delay (epochs)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of epochs that finalisation has been delayed.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "yellow", "value": 2 }, { "color": "red", "value": 5 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 15, "y": 33 }, "id": 52, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "clamp_min(min without (instance, job, alerting, client) (beacon_epoch{instance=~\"$system\"} - beacon_finalized_epoch{instance=~\"$system\"} - 2), 0)", "instant": false, "interval": "", "legendFormat": "{{network}}", "range": true, "refId": "A" } ], "title": "Finalisation Delay (epochs)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of validators which voted for the correct target in the previous epoch.", "fieldConfig": { "defaults": { "decimals": 1, "mappings": [ { "$$hashKey": "object:167", "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "#EAB839", "value": 0.67 }, { "color": "green", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 33 }, "id": 55, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "max by (network) (beacon_previous_correct_validators{instance=~\"$system\"} / beacon_previous_active_validators{instance=~\"$system\"})", "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Network Participation Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of slots the head block is behind the current slot. Empty slots may mean the head block trails by a few slots.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/.* Blocks\\/second/" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "decimals", "value": 4 }, { "id": "custom.axisLabel", "value": "blocks/second" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 36 }, "id": 66, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "clamp_min(beacon_slot{instance=~\"$system\"} - beacon_head_slot{instance=~\"$system\"}, 0)", "interval": "", "legendFormat": "{{instance}} Head Slot", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_min(min(beacon_epoch{instance=~\"$system\"} - beacon_current_justified_epoch{instance=~\"$system\"} - 1) without (instance), 0)", "interval": "", "legendFormat": "Justification Delay", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_min(min(beacon_epoch{instance=~\"$system\"} - beacon_finalized_epoch{instance=~\"$system\"} - 2) without (instance), 0)", "interval": "", "legendFormat": "Finalisation Delay", "refId": "C" } ], "title": "Trailing Distance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Overall participation rate in the chain. The Correct Validators must be above 2/3rd for the chain to finalize.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "transparent", "value": 0.66666666667 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 36 }, "id": 63, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "max by (network) (beacon_previous_live_validators{instance=~\"$system\"} / beacon_previous_active_validators{instance=~\"$system\"})", "interval": "", "legendFormat": "{{network}} Live Validators", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "max by (network) (beacon_previous_correct_validators{instance=~\"$system\"} / beacon_previous_active_validators{instance=~\"$system\"})", "interval": "", "legendFormat": "{{network}} Correct Validators", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "max by (network) (beacon_previous_live_sync_committee{instance=~\"$system\"} / (512 * 32))", "hide": false, "interval": "", "legendFormat": "{{network}} Sync Committee Participation", "refId": "D" } ], "title": "Participation Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Breakdown of client software run by peers. Note that software cannot always be determined for peers and percentage of peers isn't indicative of number of validators using that software.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 75, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "percent" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 36 }, "id": 65, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum by (exported_client, client) (libp2p_connected_peers_current{instance=~\"$system\"})", "format": "time_series", "interval": "", "legendFormat": "{{exported_client}}", "range": true, "refId": "A" } ], "title": "Peer Client Software", "type": "timeseries" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, "id": 99, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total number of prepared proposers", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "dark-blue" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 12, "x": 0, "y": 58 }, "id": 123, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(beacon_proposers_data_total{instance=~\"$system\", type=\"prepared_proposers\"})", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Prepared Proposers", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total number of registered validators willing to produce blocks using builder infrastructure", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-purple" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 12, "x": 12, "y": 58 }, "id": 124, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(beacon_proposers_data_total{instance=~\"$system\", type=\"registered_validators\"})", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Registered Validators", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "local_el" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "builder" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*fallback.*" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 61 }, "id": 100, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (source) (increase(beacon_execution_payload_source{instance=~\"$system\"}[$__range])) or sum by (source) (increase(beacon_execution_payload_source_total{instance=~\"$system\"}[$__range]))", "interval": "", "legendFormat": "{{source}}", "refId": "A" } ], "title": "Execution Payload Source", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 61 }, "id": 101, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (fallback_reason) (increase(beacon_execution_payload_source{instance=~\"$system\", source=\"builder_local_el_fallback\", fallback_reason!~\"\"}[$__range]))", "interval": "", "legendFormat": "{{fallback_reason}}", "refId": "A" } ], "title": "Execution Payload Fallback Reason", "type": "piechart" } ], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Beacon Node Block Production", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 47 }, "id": 105, "panels": [ { "gridPos": { "h": 2, "w": 12, "x": 0, "y": 59 }, "id": 111, "options": { "content": "

\n
Builder Status API
\n

", "mode": "html" }, "pluginVersion": "9.3.1", "transparent": true, "type": "text" }, { "gridPos": { "h": 2, "w": 12, "x": 12, "y": 59 }, "id": 109, "options": { "content": "

\n
Builder Validator Registration API
\n

", "mode": "html" }, "pluginVersion": "9.3.1", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Status API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "FAILED" }, "properties": [ { "id": "color", "value": { "fixedColor": "#ff0000", "mode": "fixed" } }, { "id": "custom.drawStyle", "value": "bars" }, { "id": "custom.fillOpacity", "value": 39 } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 61 }, "id": 112, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"status\"}[5m]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"status\", outcome=\"error\"}[5m]))", "hide": false, "interval": "", "legendFormat": "FAILED", "refId": "B" } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Validator Registration API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "FAILED" }, "properties": [ { "id": "color", "value": { "fixedColor": "#ff0000", "mode": "fixed" } }, { "id": "custom.drawStyle", "value": "bars" }, { "id": "custom.fillOpacity", "value": 45 } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 61 }, "id": 103, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"register_validators\"}[5m]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"register_validators\", outcome=\"error\"}[5m]))", "hide": false, "interval": "", "legendFormat": "FAILED", "refId": "B" } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Validator Registration API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 3, "x": 0, "y": 72 }, "id": 122, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "list", "placement": "right", "showLegend": false, "values": [] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"status\"}[$__range]))", "interval": "", "legendFormat": "{{outcome}}", "refId": "A" } ], "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Status API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "FAILED" }, "properties": [ { "id": "color", "value": { "fixedColor": "#ff0000", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 9, "x": 3, "y": 72 }, "id": 113, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"status\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Validator Registration API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 3, "x": 12, "y": 72 }, "id": 121, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "list", "placement": "right", "showLegend": false, "values": [] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"register_validators\"}[$__range]))", "interval": "", "legendFormat": "{{outcome}}", "refId": "A" } ], "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Validator Registration API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 9, "x": 15, "y": 72 }, "id": 102, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"register_validators\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "type": "piechart" }, { "gridPos": { "h": 2, "w": 12, "x": 0, "y": 81 }, "id": 110, "options": { "content": "

\n
Builder Get Header API
\n

", "mode": "html" }, "pluginVersion": "9.3.1", "transparent": true, "type": "text" }, { "gridPos": { "h": 2, "w": 12, "x": 12, "y": 81 }, "id": 114, "options": { "content": "

\n
Builder Get Payload API
\n

", "mode": "html" }, "pluginVersion": "9.3.1", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Get Header API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "FAILED" }, "properties": [ { "id": "custom.drawStyle", "value": "bars" }, { "id": "color", "value": { "fixedColor": "#ff0000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 46 } ] }, { "matcher": { "id": "byName", "options": "[300,500)" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "[500,1000)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "[0,100)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 83 }, "id": 107, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"get_header\"}[5m]))", "hide": false, "interval": "", "legendFormat": "{{interval}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"get_header\", outcome=\"error\"}[5m]))", "hide": false, "interval": "", "legendFormat": "FAILED", "refId": "B" } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Get Payload API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "FAILED" }, "properties": [ { "id": "custom.drawStyle", "value": "bars" }, { "id": "color", "value": { "fixedColor": "#ff0000", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 50 } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 83 }, "id": 115, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"get_payload\"}[5m]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (rate(beacon_builder_requests_total{instance=~\"$system\", method=\"get_payload\", outcome=\"error\"}[5m]))", "hide": false, "interval": "", "legendFormat": "FAILED", "refId": "B" } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Get Header API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 3, "x": 0, "y": 94 }, "id": 119, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "list", "placement": "right", "showLegend": false, "values": [] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"get_header\"}[$__range]))", "interval": "", "legendFormat": "{{outcome}}", "refId": "A" } ], "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Get Header API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 9, "x": 3, "y": 94 }, "id": 106, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"get_header\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Get Payload API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 3, "x": 12, "y": 94 }, "id": 120, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "list", "placement": "right", "showLegend": false, "values": [] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (outcome) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"get_payload\"}[$__range]))", "interval": "", "legendFormat": "{{outcome}}", "refId": "A" } ], "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Builder Get Payload API", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,300\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[300,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 9, "x": 15, "y": 94 }, "id": 116, "options": { "displayLabels": [ "name", "value" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(beacon_builder_requests_total{instance=~\"$system\", method=\"get_payload\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "type": "piechart" } ], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Beacon Node Builder API Client", "type": "row" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 48 }, "id": 95, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Block Timings", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,4000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[4000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[12000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[8000,12000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 49 }, "id": 96, "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value", "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(beacon_block_import_delay_counter{instance=~\"$system\", stage=\"arrival\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "title": "Block Arrival Delays", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Block total processing time for successfully imported blocks", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,3000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[3000,4000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[4000,5000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[5000,8000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[12000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[8000,12000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 49 }, "id": 97, "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value", "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum by (interval) (increase(beacon_block_import_delay_counter{instance=~\"$system\", stage=\"total_processing_time\", result=\"success\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "range": true, "refId": "A" } ], "title": "Block Total Processing Time", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "How long Teku waited for the execution layer to finish block processing", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*\\[0,50\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[50,100\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[1000,2000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[100,250\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[500,1000\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[250,500\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\[2000,∞\\)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 60 }, "id": 139, "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value", "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (interval) (increase(beacon_block_import_delay_counter{instance=~\"$system\", stage=\"execution_payload_result_received\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "refId": "A" } ], "title": "Wait for Block Execution Layer Processing Time", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Block import failures grouped by failure reason", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 60 }, "id": 147, "options": { "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value", "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum by (result) (increase(beacon_block_import_delay_counter{instance=~\"$system\", stage=\"total_processing_time\", result!=\"success\"}[$__range]))", "interval": "", "legendFormat": "{{interval}}", "range": true, "refId": "A" } ], "title": "Failed block import", "type": "piechart" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 71 }, "id": 77, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 11, "w": 24, "x": 0, "y": 72 }, "id": 59, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "irate(process_cpu_seconds_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} {{job}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "avg without (instance) (irate(process_cpu_seconds_total{instance=~\"$system\"}[1m]))", "hide": true, "interval": "", "legendFormat": "Average {{job}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "max without (instance) (irate(process_cpu_seconds_total{instance=~\"$system\"}[1m]))", "hide": true, "interval": "", "legendFormat": "Max {{job}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum by (instance) (irate(node_cpu_seconds_total{instance=~\"$system\",mode!=\"idle\"}[1m]))", "hide": false, "interval": "", "legendFormat": "{{instance}} Total", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "avg by (instance,mode) (irate(node_cpu_seconds_total{instance=~\"$system\",mode!=\"idle\"}[1m]))", "hide": true, "interval": "", "legendFormat": "{{instance}} {{mode}}", "refId": "E" } ], "title": "CPU usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of old gen heap used.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMax": 100, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "area" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 83 }, "id": 141, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(jvm_memory_pool_bytes_used{job=\"besu-node\", pool=\"G1 Old Gen\", instance=~\"$system\"} / jvm_memory_pool_bytes_max{job=\"besu-node\", pool=\"G1 Old Gen\", instance=~\"$system\"}) * 100 ", "hide": false, "legendFormat": "{{instance}} {{job}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(jvm_memory_pool_bytes_used{job=\"teku-node\", pool=\"G1 Old Gen\", instance=~\"$system\"} / jvm_memory_pool_bytes_max{job=\"teku-node\", pool=\"G1 Old Gen\", instance=~\"$system\"}) * 100", "hide": false, "legendFormat": "{{instance}} {{job}}", "range": true, "refId": "B" } ], "title": "Old Gen Heap Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byValue", "options": { "op": "gte", "reducer": "allIsZero", "value": 0 } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": true, "tooltip": true, "viz": false } } ] }, { "matcher": { "id": "byValue", "options": { "op": "gte", "reducer": "allIsNull", "value": 0 } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": true, "tooltip": true, "viz": false } } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 83 }, "id": 60, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "jvm_memory_pool_bytes_used{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} {{job}} {{pool}} Used", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (jvm_memory_pool_bytes_used{instance=~\"$system\"})", "hide": false, "interval": "", "legendFormat": "{{instance}} {{job}} {{pool}} Used Total", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "process_resident_memory_bytes{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} {{job}} {{pool}} Process Total", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": " (jvm_memory_pool_bytes_max{instance=~\"$system\"})", "hide": false, "interval": "", "legendFormat": "{{instance}} {{job}} {{pool}} Max", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes{instance=~\"$system\"}", "hide": true, "interval": "", "legendFormat": "{{instance}} System Memory", "refId": "E" } ], "title": "Memory Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byValue", "options": { "op": "gte", "reducer": "allIsNull", "value": 0 } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": true, "tooltip": true, "viz": false } } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 83 }, "id": 61, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "rate(jvm_gc_collection_seconds_sum{instance=~\"$system\"}[1m])", "interval": "", "legendFormat": "{{instance}} {{job}} {{gc}}", "range": true, "refId": "C" } ], "title": "GC Time", "type": "timeseries" } ], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "System", "type": "row" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 72 }, "id": 83, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Network", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "binBps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 73 }, "id": 71, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (instance) (irate(node_network_transmit_bytes_total{instance=~\"$system\"}[1m]))", "hide": false, "interval": "", "legendFormat": "{{instance}} Transmit", "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (instance) (irate(node_network_receive_bytes_total{instance=~\"$system\"}[1m]))", "hide": false, "interval": "", "legendFormat": "{{instance}} Receive", "refId": "A" } ], "title": "Network Traffic", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] }, { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "prd-clc-besu-teku-mainnet-secure-canary-01 /eth2/6a95a1a9/blob_sidecar_5/ssz_snappy" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 73 }, "id": 72, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by (instance)(irate(libp2p_gossip_messages_total{instance=~\"$system\"}[1m]))", "hide": false, "interval": "", "legendFormat": "{{instance}} Total", "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "irate(libp2p_gossip_messages_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} {{topic}}", "refId": "A" } ], "title": "Gossip Messages", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 73 }, "id": 133, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "irate(etwork_rpc_blocks_by_root_requested_blocks_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} Blocks By Root", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "irate(network_rpc_blocks_by_range_requested_blocks_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} Blocks By Range", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "irate(network_rpc_blob_sidecars_by_range_requested_sidecars_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} Blob sidecars By Range", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "irate(network_rpc_blob_sidecars_by_root_requested_blob_sidecars_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} Blob sidecars By Root", "range": true, "refId": "C" } ], "title": "RPC Blocks and Blob sidecars Requested", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "links": [], "mappings": [], "min": 0, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 83 }, "id": 134, "options": { "displayLabels": [], "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "values": [ "value", "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum by (reason) (increase(network_peer_goodbye_total{instance=~\"$system\"}[$__range]))", "hide": false, "interval": "", "legendFormat": "{{reason}}", "range": true, "refId": "J" } ], "title": "Peer Disconnect Reasons", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "links": [], "mappings": [], "min": 0, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 83 }, "id": 135, "options": { "displayLabels": [], "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "values": [ "value", "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum by (status) (increase(network_peer_chain_validation_attempts{instance=~\"$system\",status!=\"started\"}[$__range])) or sum by (status) (increase(network_peer_chain_validation_attempts_total{instance=~\"$system\",status!=\"started\"}[$__range]))", "hide": false, "interval": "", "legendFormat": "{{status}}", "range": true, "refId": "J" } ], "title": "Peer Chain Validation Attempts", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "area" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "green", "value": 2 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 83 }, "id": 144, "options": { "legend": { "calcs": [ "min", "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Min", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "network_subnet_peer_count{instance=~\"$system\", subnet=~\"attestation.*\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} {{subnet}}", "range": true, "refId": "J" } ], "title": "Peer Count By Subnet", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 93 }, "id": 130, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 42 }, "id": 128, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "1 - node_filesystem_avail_bytes{instance=~\"$system\",mountpoint=~\"(/data|/)\"} / node_filesystem_size_bytes{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} {{mountpoint}}", "range": true, "refId": "J" } ], "title": "Disk Space Used", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 42 }, "id": 131, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$system\",device=\"nvme1n1\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} {{device}} IO Time", "refId": "J" } ], "title": "Disk Active Time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 42 }, "id": 132, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$system\"}[1m]) / irate(node_disk_reads_completed_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} {{device}} Read Latency", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$system\"}[1m]) / irate(node_disk_writes_completed_total{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} {{device}} Write Latency", "range": true, "refId": "A" } ], "title": "Disk Latency", "type": "timeseries" } ], "title": "Disk", "type": "row" }, { "collapsed": false, "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 94 }, "id": 85, "panels": [], "targets": [ { "datasource": { "type": "prometheus", "uid": "ljwa6Go4k" }, "refId": "A" } ], "title": "Queues, Excecutors and Caches", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Tracks number of signatures waiting to be batch verified", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 95 }, "id": 68, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "executor_signature_verifications_queue_size{instance=~\"$system\"}", "interval": "", "legendFormat": "{{instance}}", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "avg without (instance) (executor_signature_verifications_queue_size{instance=~\"$system\"})", "hide": false, "interval": "", "legendFormat": "Average", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "max without (instance) (executor_signature_verifications_queue_size{instance=~\"$system\"})", "hide": false, "interval": "", "legendFormat": "Max", "refId": "B" } ], "title": "Batch Signature Verification Queue Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 95 }, "id": 69, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "irate(executor_signature_verifications_task_count{instance=~\"$system\"}[1m]) / irate(executor_signature_verifications_batch_count{instance=~\"$system\"}[1m]) or irate(executor_signature_verifications_task_count_total{instance=~\"$system\"}[1m]) / irate(executor_signature_verifications_batch_count_total{instance=~\"$system\"}[1m])", "interval": "", "legendFormat": "{{instance}}", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "avg without (instance) (irate(executor_signature_verifications_task_count{instance=~\"$system\"}[1m]) / irate(executor_signature_verifications_batch_count{instance=~\"$system\"}[1m])) or avg without (instance) (irate(executor_signature_verifications_task_count_total{instance=~\"$system\"}[1m]) / irate(executor_signature_verifications_batch_count_total{instance=~\"$system\"}[1m]))", "hide": false, "interval": "", "legendFormat": "Average", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "max without (instance) (irate(executor_signature_verifications_task_count{instance=~\"$system\"}[1m]) / irate(executor_signature_verifications_batch_count{instance=~\"$system\"}[1m])) or max without (instance) (irate(executor_signature_verifications_task_count_total{instance=~\"$system\"}[1m]) / irate(executor_signature_verifications_batch_count_total{instance=~\"$system\"}[1m]))", "hide": false, "interval": "", "legendFormat": "Max", "refId": "B" } ], "title": "Signature Verification Batch Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 95 }, "id": 70, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "executor_signature_verifications_batch_size{instance=~\"$system\"}", "interval": "", "legendFormat": "{{instance}} {{quantile}}", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "avg without (instance) (executor_signature_verifications_batch_size{instance=~\"$system\"})", "hide": false, "interval": "", "legendFormat": "Average {{quantile}}", "refId": "A" } ], "title": "Signature Verification Batch Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 105 }, "id": 74, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "beacon_pending_pool_size{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} Pending {{type}}", "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "beacon_future_items_size{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} Future {{type}}", "refId": "A" } ], "title": "Pending Pool Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 105 }, "id": 75, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "executor_queue_size{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} {{name}}", "refId": "J" } ], "title": "Executor Queues", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 105 }, "id": 73, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "storage_memory_states_cache_size{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} States", "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "storage_memory_block_count{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} Blocks", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "storage_memory_checkpoint_states_cache_size{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} Checkpoint States", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "storage_memory_epoch_states_cache_size{instance=~\"$system\"}", "hide": false, "instant": false, "legendFormat": "{{instance}} Epoch States", "range": true, "refId": "C" } ], "title": "In Memory Storage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 115 }, "id": 127, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "eventbus_event_published_count{instance=~\"$system\"} - ignoring(subscriber) group_right eventbus_event_consumed_count{instance=~\"$system\"} or eventbus_event_published_count_total{instance=~\"$system\"} - ignoring(subscriber) group_right eventbus_event_consumed_count_total{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} {{channel}} {{subscriber}}", "refId": "J" } ], "title": "Event Channels", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*GC" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 8, "y": 115 }, "id": 136, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "beacon_ee_request_queue_size{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} EE Queue Size", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "beacon_eth1_request_queue_size{instance=~\"$system\"} or beacon_eth1_request_queue_size_total{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} Eth1 Queue Size", "range": true, "refId": "A" } ], "title": "EL Request Queues", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".* Blocks / Minute" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 115 }, "id": 138, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "beacon_historical_block_sync_earliest_block{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} earliest available block", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "-1 * delta(beacon_historical_block_sync_earliest_block{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} Blocks / Minute", "range": true, "refId": "A" } ], "title": "Historic Block Sync", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Shows the number of BlsToExecutionChange operations in the operation pool.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".* Blocks / Minute" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] } ] }, "gridPos": { "h": 10, "w": 8, "x": 0, "y": 125 }, "id": 146, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Max", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "beacon_operation_pool_size_SignedBlsToExecutionChangePool{instance=~\"$system\"}", "hide": false, "interval": "", "legendFormat": "{{instance}} execution changes", "range": true, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "-1 * delta(beacon_operation_pool_size_SignedBlsToExecutionChangePool{instance=~\"$system\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}} execution changes / Minute", "range": true, "refId": "A" } ], "title": "BLS change queue", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 135 }, "id": 148, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 14, "w": 12, "x": 0, "y": 1 }, "id": 149, "options": { "legend": { "calcs": [ "firstNotNull", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "beacon_block_blobs_trackers_pool_size{instance=~\"$system\"} or beacon_block_blobs_trackers_pool_size_total{instance=~\"$system\"}", "instant": false, "legendFormat": "{{instance}} {{type}}", "range": true, "refId": "A" } ], "title": "Pooled Sidecars and Trackers", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 14, "w": 12, "x": 12, "y": 1 }, "id": 150, "options": { "legend": { "calcs": [ "firstNotNull", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(beacon_block_blobs_trackers_pool_stats{instance=~\"$system\"}[$__rate_interval]) or rate(beacon_block_blobs_trackers_pool_stats_total{instance=~\"$system\"}[$__rate_interval])", "instant": false, "legendFormat": "{{instance}} {{type}} {{subtype}}", "range": true, "refId": "A" } ], "title": "Trackers pool stats counters", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 19, "w": 8, "x": 0, "y": 15 }, "id": 151, "options": { "legend": { "calcs": [ "firstNotNull", "mean", "lastNotNull", "diff" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "beacon_block_blobs_trackers_pool_stats{instance=~\"$system\", subtype=\"rpc_fetch\"} or beacon_block_blobs_trackers_pool_stats_total{instance=~\"$system\", subtype=\"rpc_fetch\"}", "instant": false, "legendFormat": "{{instance}} {{type}}", "range": true, "refId": "A" } ], "title": "RPC fetches", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 19, "w": 7, "x": 8, "y": 15 }, "id": 152, "options": { "legend": { "calcs": [ "firstNotNull", "mean", "lastNotNull", "diff" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "beacon_block_blobs_trackers_pool_stats{instance=~\"$system\", subtype=\"rpc\"} or beacon_block_blobs_trackers_pool_stats_total{instance=~\"$system\", subtype=\"rpc\"}", "instant": false, "legendFormat": "{{instance}} {{type}}", "range": true, "refId": "A" } ], "title": "RPC receives", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 19, "w": 9, "x": 15, "y": 15 }, "id": 153, "options": { "legend": { "calcs": [ "firstNotNull", "mean", "lastNotNull", "diff" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "beacon_block_blobs_trackers_pool_stats{instance=~\"$system\", subtype=~\"gossip_duplicate|rpc_duplicate\"} or beacon_block_blobs_trackers_pool_stats_total{instance=~\"$system\", subtype=~\"gossip_duplicate|rpc_duplicate\"}", "instant": false, "legendFormat": "{{instance}} {{type}} {{subtype}}", "range": true, "refId": "A" } ], "title": "Duplicates", "type": "timeseries" } ], "title": "Block and Blob-Sidecars trackers Pool", "type": "row" } ], "refresh": "30s", "revision": 1, "schemaVersion": 39, "tags": [ "teku", "ethereum", "eth2.0" ], "templating": { "list": [ { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "validator_local_validator_count{network=~\"$network\",alerting=~\"$alerting\"}", "hide": 0, "includeAll": true, "label": "System", "multi": true, "name": "system", "options": [], "query": { "query": "validator_local_validator_count{network=~\"$network\",alerting=~\"$alerting\"}", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "/instance=\"([^\"]*)\"/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(network)", "hide": 0, "includeAll": true, "label": "Network", "multi": false, "name": "network", "options": [], "query": { "query": "label_values(network)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(alerting)", "hide": 0, "includeAll": true, "label": "Alerting", "multi": true, "name": "alerting", "options": [], "query": { "query": "label_values(alerting)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "Teku - Detailed", "uid": "G_GUWeJ4z", "version": 54, "weekStart": "" }