diff --git a/prometheus_grafana_loki/dashboards/docker_containers.json b/prometheus_grafana_loki/dashboards/docker_containers.json
deleted file mode 100644
index 9589e7c..0000000
--- a/prometheus_grafana_loki/dashboards/docker_containers.json
+++ /dev/null
@@ -1,1270 +0,0 @@
-{
- "id": null,
- "title": "Docker Containers",
- "description": "Containers metrics",
- "tags": [
- "docker"
- ],
- "style": "dark",
- "timezone": "browser",
- "editable": true,
- "hideControls": false,
- "sharedCrosshair": true,
- "rows": [
- {
- "collapse": false,
- "editable": true,
- "height": "150px",
- "panels": [
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(50, 172, 45, 0.97)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(245, 54, 54, 0.9)"
- ],
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "format": "percent",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": true,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 4,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "sum(rate(container_cpu_user_seconds_total{image!=\"\"}[1m])) / count(node_cpu_seconds_total{mode=\"user\"}) * 100",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": "65, 90",
- "title": "CPU Load",
- "transparent": false,
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 7,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "machine_cpu_cores",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "",
- "metric": "machine_cpu_cores",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": "",
- "title": "CPU Cores",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg"
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(50, 172, 45, 0.97)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(245, 54, 54, 0.9)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "percent",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": true,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 5,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "(sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100",
- "interval": "10s",
- "intervalFactor": 2,
- "legendFormat": "",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": "65, 90",
- "title": "Memory Load",
- "transparent": false,
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "format": "bytes",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 2,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "sum(container_memory_usage_bytes{image!=\"\"})",
- "interval": "10s",
- "intervalFactor": 2,
- "legendFormat": "",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": "",
- "timeFrom": "10s",
- "title": "Used Memory",
- "transparent": false,
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(50, 172, 45, 0.97)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(245, 54, 54, 0.9)"
- ],
- "datasource": "Prometheus",
- "decimals": null,
- "editable": true,
- "error": false,
- "format": "percent",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": true,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 6,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "(node_filesystem_size_bytes{fstype=\"ext4\"} - node_filesystem_free_bytes{fstype=\"ext4\"}) / node_filesystem_size_bytes{fstype=\"ext4\"} * 100",
- "interval": "30s",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 30
- }
- ],
- "thresholds": "65, 90",
- "title": "Storage Load",
- "transparent": false,
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "format": "bytes",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 3,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "sum(container_fs_usage_bytes)",
- "interval": "30s",
- "intervalFactor": 2,
- "refId": "A",
- "step": 60
- }
- ],
- "thresholds": "",
- "title": "Used Storage",
- "transparent": false,
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- }
- ],
- "title": "Overview"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "150px",
- "panels": [
- {
- "aliasColors": {},
- "bars": true,
- "datasource": "Prometheus",
- "decimals": 0,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)",
- "thresholdLine": false
- },
- "id": 9,
- "isNew": true,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "scalar(count(container_memory_usage_bytes{image!=\"\"}) > 0)",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "containers",
- "refId": "A",
- "step": 2
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Running Containers",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "none",
- "label": "",
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": true,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 10,
- "isNew": true,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "load 1m",
- "color": "#BF1B00"
- }
- ],
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "node_load1",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "load 1m",
- "metric": "node_load1",
- "refId": "A",
- "step": 2
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "System Load",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 15,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": false,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "read",
- "yaxis": 1
- },
- {
- "alias": "written",
- "yaxis": 1
- },
- {
- "alias": "io time",
- "yaxis": 2
- }
- ],
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(irate(node_disk_read_bytes_total[5m]))",
- "interval": "2s",
- "intervalFactor": 4,
- "legendFormat": "read",
- "metric": "",
- "refId": "A",
- "step": 8
- },
- {
- "expr": "sum(irate(node_disk_written_bytes_total[5m]))",
- "interval": "2s",
- "intervalFactor": 4,
- "legendFormat": "written",
- "metric": "",
- "refId": "B",
- "step": 8
- },
- {
- "expr": "sum(irate(node_disk_io_time_seconds_total[5m]))",
- "interval": "2s",
- "intervalFactor": 4,
- "legendFormat": "io time",
- "metric": "",
- "refId": "C",
- "step": 8
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "I/O Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- }
- ],
- "title": "Host stats"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 8,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m])) / scalar(count(node_cpu_seconds_total{mode=\"user\"})) * 100",
- "intervalFactor": 10,
- "legendFormat": "{{ name }}",
- "metric": "container_cpu_user_seconds_total",
- "refId": "A",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Container CPU Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "percent",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "title": "CPU"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 11,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (name)(container_memory_usage_bytes{image!=\"\",container_label_org_label_schema_group=\"\"})",
- "intervalFactor": 1,
- "legendFormat": "{{ name }}",
- "metric": "container_memory_usage",
- "refId": "A",
- "step": 1
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Container Memory Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 12,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (name) (container_memory_cache{image!=\"\",container_label_org_label_schema_group=\"\"})",
- "intervalFactor": 2,
- "legendFormat": "{{name}}",
- "metric": "container_memory_cache",
- "refId": "A",
- "step": 2
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Container Cached Memory Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "title": "Memory"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 13,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (name) (rate(container_network_receive_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))",
- "intervalFactor": 10,
- "legendFormat": "{{ name }}",
- "metric": "container_network_receive_bytes_total",
- "refId": "A",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Container Network Input",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 14,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (name) (rate(container_network_transmit_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))",
- "intervalFactor": 10,
- "legendFormat": "{{ name }}",
- "metric": "container_network_transmit_bytes_total",
- "refId": "A",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Container Network Output",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "title": "Network"
- }
- ],
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "templating": {
- "list": []
- },
- "annotations": {
- "list": []
- },
- "refresh": "10s",
- "schemaVersion": 12,
- "version": 8,
- "links": [],
- "gnetId": null
-}
diff --git a/prometheus_grafana_loki/dashboards/docker_host.json b/prometheus_grafana_loki/dashboards/docker_host.json
index a6beef7..82195f3 100644
--- a/prometheus_grafana_loki/dashboards/docker_host.json
+++ b/prometheus_grafana_loki/dashboards/docker_host.json
@@ -1,1409 +1,2227 @@
{
- "id": null,
- "title": "Docker Host",
- "description": "Docker host metrics",
- "tags": [
- "system"
- ],
- "style": "dark",
- "timezone": "browser",
- "editable": true,
- "hideControls": false,
- "sharedCrosshair": true,
- "rows": [
+ "annotations": [
{
- "collapse": false,
- "editable": true,
- "height": "100px",
- "panels": [
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "decimals": 1,
- "editable": true,
- "error": false,
- "format": "s",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
+ "kind": "AnnotationQuery",
+ "spec": {
+ "builtIn": true,
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "query": {
+ "datasource": {
+ "name": "-- Grafana --"
},
- "id": 1,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "s",
- "postfixFontSize": "80%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "node_time_seconds - node_boot_time_seconds",
- "interval": "30s",
- "intervalFactor": 1,
- "refId": "A",
- "step": 30
- }
- ],
- "thresholds": "",
- "title": "Uptime",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "percent",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 13,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "sum(rate(node_cpu_seconds_total{mode=\"idle\"}[1m])) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"}))",
- "interval": "10s",
- "intervalFactor": 2,
- "legendFormat": "",
- "refId": "A",
- "step": 20
- }
- ],
- "thresholds": "",
- "title": "CPU Idle",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 12,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "machine_cpu_cores",
- "intervalFactor": 2,
- "metric": "machine_cpu_cores",
- "refId": "A",
- "step": 2
- }
- ],
- "thresholds": "",
- "title": "CPU Cores",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "bytes",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 2,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "node_memory_MemAvailable_bytes",
- "interval": "30s",
- "intervalFactor": 2,
- "legendFormat": "",
- "refId": "A",
- "step": 60
- }
- ],
- "thresholds": "",
- "title": "Available Memory",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "bytes",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 3,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "node_memory_SwapFree_bytes",
- "interval": "30s",
- "intervalFactor": 2,
- "refId": "A",
- "step": 60
- }
- ],
- "thresholds": "",
- "title": "Free Swap",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "bytes",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "id": 4,
- "interval": null,
- "isNew": true,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "span": 2,
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "targets": [
- {
- "expr": "sum(node_filesystem_free_bytes{fstype=\"ext4\"})",
- "interval": "30s",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 30
- }
- ],
- "thresholds": "",
- "title": "Free Storage",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg",
- "timeFrom": "10s",
- "hideTimeOverride": true
+ "group": "grafana",
+ "kind": "DataQuery",
+ "spec": {},
+ "version": "v0"
}
- ],
- "title": "Available resources"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "150px",
- "panels": [
- {
- "aliasColors": {},
- "bars": true,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 9,
- "isNew": true,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "load 1m",
- "color": "#1F78C1"
- }
- ],
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "node_load1",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "load 1m",
- "refId": "A",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Load Average 1m",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": true,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 10,
- "isNew": true,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "blocked by I/O",
- "color": "#58140C"
- }
- ],
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "node_procs_running",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "running",
- "metric": "node_procs_running",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "node_procs_blocked",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "blocked by I/O",
- "metric": "node_procs_blocked",
- "refId": "B",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Processes",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": true,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 11,
- "isNew": true,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "interrupts",
- "color": "#806EB7"
- }
- ],
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": " irate(node_intr_total[5m])",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "interrupts",
- "metric": "node_intr_total",
- "refId": "A",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Interrupts",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "title": "Load"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 4,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 5,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 12,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(node_cpu_seconds_total[1m])) by (mode) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"}))",
- "intervalFactor": 10,
- "legendFormat": "{{ mode }}",
- "metric": "node_cpu_seconds_total",
- "refId": "A",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "percent",
- "label": null,
- "logBase": 1,
- "max": 100,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- }
- ]
- }
- ],
- "title": "CPU"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 4,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 6,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Used",
- "color": "#BF1B00"
- },
- {
- "alias": "Free",
- "color": "#7EB26D"
- },
- {
- "alias": "Buffers",
- "color": "#6ED0E0"
- },
- {
- "alias": "Cached",
- "color": "#EF843C"
- }
- ],
- "span": 12,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)",
- "intervalFactor": 1,
- "legendFormat": "Used",
- "refId": "A",
- "step": 1
- },
- {
- "expr": "node_memory_MemFree_bytes",
- "intervalFactor": 1,
- "legendFormat": "Free",
- "refId": "B",
- "step": 1
- },
- {
- "expr": "node_memory_Buffers_bytes",
- "intervalFactor": 1,
- "legendFormat": "Buffers",
- "refId": "C",
- "step": 1
- },
- {
- "expr": "node_memory_Cached_bytes",
- "intervalFactor": 1,
- "legendFormat": "Cached",
- "refId": "D",
- "step": 1
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- }
- ],
- "title": "Memory"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 7,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "read",
- "yaxis": 1
- },
- {
- "alias": "written",
- "yaxis": 1
- },
- {
- "alias": "io time",
- "yaxis": 2
- }
- ],
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(irate(node_disk_read_bytes_total[1m]))",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "read",
- "metric": "node_disk_read_bytes_total",
- "refId": "A",
- "step": 1
- },
- {
- "expr": "sum(irate(node_disk_written_bytes_total[1m]))",
- "intervalFactor": 1,
- "legendFormat": "written",
- "metric": "node_disk_written_bytes_total",
- "refId": "B",
- "step": 1
- },
- {
- "expr": "sum(irate(node_disk_io_time_seconds_total[1m]))",
- "intervalFactor": 1,
- "legendFormat": "io time",
- "metric": "node_disk_io_time_seconds_total",
- "refId": "C",
- "step": 1
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "I/O Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- }
- ],
- "title": "I/O"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 4,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 8,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 12,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(container_network_receive_bytes_total{id=\"/\",interface!~\"br.*|lo\"}[1m]))",
- "intervalFactor": 1,
- "legendFormat": "In: {{ device }}",
- "metric": "node_network_receive_bytes_total",
- "refId": "A",
- "step": 1
- },
- {
- "expr": "sum(rate(container_network_transmit_bytes_total{id=\"/\",interface!~\"br.*|lo\"}[1m]))",
- "intervalFactor": 1,
- "legendFormat": "Out: {{ device }}",
- "metric": "node_network_transmit_bytes_total",
- "refId": "B",
- "step": 1
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Network Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "title": "Network"
- },
- {
- "collapse": false,
- "editable": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 4,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 14,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": false,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Used",
- "color": "#890F02"
- },
- {
- "alias": "Free",
- "color": "#7EB26D"
- }
- ],
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "Used",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "node_memory_SwapFree_bytes",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "Free",
- "refId": "B",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Swap Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {},
- "bars": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {
- "threshold1": null,
- "threshold1Color": "rgba(216, 200, 27, 0.27)",
- "threshold2": null,
- "threshold2Color": "rgba(234, 112, 112, 0.22)"
- },
- "id": 15,
- "isNew": true,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(node_vmstat_pswpin[1m]) * 4096 or irate(node_vmstat_pswpin[5m]) * 4096",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "In",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "rate(node_vmstat_pswpout[1m]) * 4096 or irate(node_vmstat_pswpout[5m]) * 4096",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "Out",
- "refId": "B",
- "step": 10
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Swap I/O",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "show": true
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "title": "New row"
+ }
}
],
- "time": {
- "from": "now-1h",
- "to": "now"
+ "cursorSync": "Crosshair",
+ "editable": true,
+ "elements": {
+ "panel-1": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {
+ "interval": "30s"
+ },
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 1,
+ "links": [],
+ "title": "Host CPU",
+ "vizConfig": {
+ "group": "stat",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "percentage",
+ "steps": [
+ {
+ "color": "",
+ "value": 0
+ },
+ {
+ "color": "",
+ "value": 65
+ },
+ {
+ "color": "",
+ "value": 90
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-10": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "sum by (name) (rate(container_network_transmit_bytes_total{image!=\"\",name=~\"$container\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {
+ "interval": "30s"
+ },
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 10,
+ "links": [],
+ "title": "Container Network TX",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-11": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "sum by (name) (container_fs_usage_bytes{image!=\"\",name=~\"$container\"})",
+ "format": "table",
+ "instant": true
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": [
+ {
+ "group": "organize",
+ "kind": "Transformation",
+ "spec": {
+ "options": {
+ "excludeByName": {
+ "Time": true
+ },
+ "indexByName": {
+ "Value": 1,
+ "name": 0
+ },
+ "renameByName": {
+ "Value": "Used",
+ "name": "Container"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "description": "",
+ "id": 11,
+ "links": [],
+ "title": "Container Filesystem Usage",
+ "vizConfig": {
+ "group": "table",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "footer": {
+ "reducers": []
+ },
+ "inspect": false,
+ "wrapText": false
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "cellHeight": "sm",
+ "enablePagination": false,
+ "showHeader": false,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Used"
+ }
+ ]
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-12": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "editorMode": "code",
+ "expr": "sum by (device) (rate(node_disk_read_bytes_total{device!~\"^(loop|ram|fd|sr).*\"}[$__rate_interval]))",
+ "legendFormat": "Read - {{device}}",
+ "range": true
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "editorMode": "code",
+ "expr": "sum by (device) (rate(node_disk_written_bytes_total{device!~\"^(loop|ram|fd|sr).*\"}[$__rate_interval]))",
+ "legendFormat": "Write - {{device}}",
+ "range": true
+ },
+ "version": "v0"
+ },
+ "refId": "B"
+ }
+ }
+ ],
+ "queryOptions": {
+ "interval": "30s"
+ },
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 12,
+ "links": [],
+ "title": "Local Disk I/O",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-13": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|docker.*|br-.*|veth.*\"}[$__rate_interval]))",
+ "legendFormat": "RX"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|docker.*|br-.*|veth.*\"}[$__rate_interval]))",
+ "legendFormat": "TX"
+ },
+ "version": "v0"
+ },
+ "refId": "B"
+ }
+ }
+ ],
+ "queryOptions": {
+ "interval": "30s"
+ },
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 13,
+ "links": [],
+ "title": "Host Network",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-14": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "node_load1",
+ "legendFormat": "{{instance}} load 1m"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "node_load5",
+ "legendFormat": "{{instance}} load 5m"
+ },
+ "version": "v0"
+ },
+ "refId": "B"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "node_load15",
+ "legendFormat": "{{instance}} load 15m"
+ },
+ "version": "v0"
+ },
+ "refId": "C"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 14,
+ "links": [],
+ "title": "System Load",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-15": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "editorMode": "code",
+ "expr": "time() - node_boot_time_seconds",
+ "range": true
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 15,
+ "links": [],
+ "title": "Host Uptime",
+ "vizConfig": {
+ "group": "stat",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "#EAB839",
+ "value": 2592000
+ },
+ {
+ "color": "red",
+ "value": 5184000
+ }
+ ]
+ },
+ "unit": "dtdurations"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-17": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "(node_filesystem_size_bytes{fstype!~\"tmpfs|devtmpfs|overlay|squashfs|ramfs|iso9660|proc|sysfs|cgroup2?|nsfs|autofs|bpf|hugetlbfs|debugfs|tracefs|configfs|fusectl|mqueue|pstore|securityfs|devpts\",mountpoint!~\"/boot.*|/run.*|/dev.*|/sys.*|/proc.*|/var/lib/docker/.*|/snap/.*|/var/lib/kubelet/.*\"} - node_filesystem_avail_bytes{fstype!~\"tmpfs|devtmpfs|overlay|squashfs|ramfs|iso9660|proc|sysfs|cgroup2?|nsfs|autofs|bpf|hugetlbfs|debugfs|tracefs|configfs|fusectl|mqueue|pstore|securityfs|devpts\",mountpoint!~\"/boot.*|/run.*|/dev.*|/sys.*|/proc.*|/var/lib/docker/.*|/snap/.*|/var/lib/kubelet/.*\"})",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "__auto",
+ "range": false
+ },
+ "version": "v0"
+ },
+ "refId": "used"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "node_filesystem_size_bytes{fstype!~\"tmpfs|devtmpfs|overlay|squashfs|ramfs|iso9660|proc|sysfs|cgroup2?|nsfs|autofs|bpf|hugetlbfs|debugfs|tracefs|configfs|fusectl|mqueue|pstore|securityfs|devpts\",mountpoint!~\"/boot.*|/run.*|/dev.*|/sys.*|/proc.*|/var/lib/docker/.*|/snap/.*|/var/lib/kubelet/.*\"}",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "__auto",
+ "range": false
+ },
+ "version": "v0"
+ },
+ "refId": "capacity"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "node_filesystem_avail_bytes{fstype!~\"tmpfs|devtmpfs|overlay|squashfs|ramfs|iso9660|proc|sysfs|cgroup2?|nsfs|autofs|bpf|hugetlbfs|debugfs|tracefs|configfs|fusectl|mqueue|pstore|securityfs|devpts\",mountpoint!~\"/boot.*|/run.*|/dev.*|/sys.*|/proc.*|/var/lib/docker/.*|/snap/.*|/var/lib/kubelet/.*\"}",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "__auto",
+ "range": false
+ },
+ "version": "v0"
+ },
+ "refId": "free"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "(1 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|devtmpfs|overlay|squashfs|ramfs|iso9660|proc|sysfs|cgroup2?|nsfs|autofs|bpf|hugetlbfs|debugfs|tracefs|configfs|fusectl|mqueue|pstore|securityfs|devpts\",mountpoint!~\"/boot.*|/run.*|/dev.*|/sys.*|/proc.*|/var/lib/docker/.*|/snap/.*|/var/lib/kubelet/.*\"}) / (node_filesystem_size_bytes{fstype!~\"tmpfs|devtmpfs|overlay|squashfs|ramfs|iso9660|proc|sysfs|cgroup2?|nsfs|autofs|bpf|hugetlbfs|debugfs|tracefs|configfs|fusectl|mqueue|pstore|securityfs|devpts\",mountpoint!~\"/boot.*|/run.*|/dev.*|/sys.*|/proc.*|/var/lib/docker/.*|/snap/.*|/var/lib/kubelet/.*\"})) * 100",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "__auto",
+ "range": false
+ },
+ "version": "v0"
+ },
+ "refId": "percent"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": [
+ {
+ "group": "joinByField",
+ "kind": "Transformation",
+ "spec": {
+ "options": {
+ "byField": "mountpoint",
+ "mode": "outer"
+ }
+ }
+ },
+ {
+ "group": "organize",
+ "kind": "Transformation",
+ "spec": {
+ "options": {
+ "excludeByName": {
+ "Time 1": true,
+ "Time 2": true,
+ "Time 3": true,
+ "Time 4": true,
+ "__name__": true,
+ "__name__ 1": true,
+ "__name__ 2": true,
+ "device 2": true,
+ "device 3": true,
+ "device 4": true,
+ "fstype 2": true,
+ "fstype 3": true,
+ "fstype 4": true,
+ "instance 1": true,
+ "instance 2": true,
+ "instance 3": true,
+ "instance 4": true,
+ "job 1": true,
+ "job 2": true,
+ "job 3": true,
+ "job 4": true
+ },
+ "includeByName": {},
+ "indexByName": {
+ "Time 1": 7,
+ "Time 2": 10,
+ "Time 3": 16,
+ "Time 4": 22,
+ "Value #capacity": 6,
+ "Value #free": 4,
+ "Value #percent": 3,
+ "Value #used": 5,
+ "__name__ 1": 15,
+ "__name__ 2": 17,
+ "device 1": 1,
+ "device 2": 11,
+ "device 3": 18,
+ "device 4": 23,
+ "fstype 1": 2,
+ "fstype 2": 12,
+ "fstype 3": 19,
+ "fstype 4": 24,
+ "instance 1": 8,
+ "instance 2": 13,
+ "instance 3": 20,
+ "instance 4": 25,
+ "job 1": 9,
+ "job 2": 14,
+ "job 3": 21,
+ "job 4": 26,
+ "mountpoint": 0
+ },
+ "renameByName": {
+ "Value #capacity": "Total",
+ "Value #free": "Free",
+ "Value #percent": "Used %",
+ "Value #used": "Used",
+ "fstype 1": ""
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "description": "",
+ "id": 17,
+ "links": [],
+ "title": "Disk Usage",
+ "vizConfig": {
+ "group": "table",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "footer": {
+ "reducers": []
+ },
+ "inspect": false
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used",
+ "scope": "series"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total",
+ "scope": "series"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Free",
+ "scope": "series"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used %",
+ "scope": "series"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "percent"
+ },
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "mode": "basic",
+ "type": "gauge",
+ "valueDisplayMode": "color"
+ }
+ },
+ {
+ "id": "min",
+ "value": 0
+ },
+ {
+ "id": "max",
+ "value": 100
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "cellHeight": "sm",
+ "frameIndex": 0,
+ "showHeader": true
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-2": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 2,
+ "links": [],
+ "title": "Host Memory",
+ "vizConfig": {
+ "group": "stat",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 1,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-4": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "count(container_last_seen{image!=\"\"})"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 4,
+ "links": [],
+ "title": "Running Containers",
+ "vizConfig": {
+ "group": "stat",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "decimals": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-5": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))",
+ "legendFormat": "{{instance}}"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {
+ "interval": "30s"
+ },
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 5,
+ "links": [],
+ "title": "Host CPU Usage",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-6": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
+ "legendFormat": "{{instance}}"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ },
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "node_memory_MemTotal_bytes",
+ "legendFormat": "Total RAM"
+ },
+ "version": "v0"
+ },
+ "refId": "B"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 6,
+ "links": [],
+ "title": "Host Memory Usage",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total RAM"
+ },
+ "properties": [
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "dash": [
+ 10,
+ 10
+ ],
+ "fill": "dash"
+ }
+ },
+ {
+ "id": "custom.fillOpacity",
+ "value": 0
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 1
+ },
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "semi-dark-red",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-7": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "100 * sum by (name) (rate(container_cpu_usage_seconds_total{image!=\"\",name=~\"$container\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {
+ "interval": "30s"
+ },
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 7,
+ "links": [],
+ "title": "Container CPU Usage",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "right",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-8": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "sum by (name) (container_memory_working_set_bytes{image!=\"\",name=~\"$container\"})",
+ "legendFormat": "{{name}}"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {},
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 8,
+ "links": [],
+ "title": "Container Memory Usage",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "right",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ },
+ "panel-9": {
+ "kind": "Panel",
+ "spec": {
+ "data": {
+ "kind": "QueryGroup",
+ "spec": {
+ "queries": [
+ {
+ "kind": "PanelQuery",
+ "spec": {
+ "hidden": false,
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "expr": "sum by (name) (rate(container_network_receive_bytes_total{image!=\"\",name=~\"$container\"}[$__rate_interval]))",
+ "legendFormat": "{{name}}"
+ },
+ "version": "v0"
+ },
+ "refId": "A"
+ }
+ }
+ ],
+ "queryOptions": {
+ "interval": "30s"
+ },
+ "transformations": []
+ }
+ },
+ "description": "",
+ "id": 9,
+ "links": [],
+ "title": "Container Network RX",
+ "vizConfig": {
+ "group": "timeseries",
+ "kind": "VizConfig",
+ "spec": {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "enableFacetedFilter": false,
+ "overflow": "ellipsis",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ }
+ },
+ "version": "13.1.3"
+ }
+ }
+ }
},
- "timepicker": {
- "refresh_intervals": [
+ "layout": {
+ "kind": "GridLayout",
+ "spec": {
+ "items": [
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-15"
+ },
+ "height": 4,
+ "width": 6,
+ "x": 0,
+ "y": 0
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-1"
+ },
+ "height": 4,
+ "width": 6,
+ "x": 6,
+ "y": 0
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-2"
+ },
+ "height": 4,
+ "width": 6,
+ "x": 12,
+ "y": 0
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-4"
+ },
+ "height": 4,
+ "width": 6,
+ "x": 18,
+ "y": 0
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-17"
+ },
+ "height": 7,
+ "width": 16,
+ "x": 0,
+ "y": 4
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-12"
+ },
+ "height": 7,
+ "width": 8,
+ "x": 16,
+ "y": 4
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-5"
+ },
+ "height": 7,
+ "width": 8,
+ "x": 0,
+ "y": 11
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-6"
+ },
+ "height": 7,
+ "width": 8,
+ "x": 8,
+ "y": 11
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-14"
+ },
+ "height": 7,
+ "width": 8,
+ "x": 16,
+ "y": 11
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-7"
+ },
+ "height": 6,
+ "width": 12,
+ "x": 0,
+ "y": 18
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-8"
+ },
+ "height": 6,
+ "width": 12,
+ "x": 12,
+ "y": 18
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-9"
+ },
+ "height": 7,
+ "width": 12,
+ "x": 0,
+ "y": 24
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-10"
+ },
+ "height": 7,
+ "width": 12,
+ "x": 12,
+ "y": 24
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-13"
+ },
+ "height": 7,
+ "width": 24,
+ "x": 0,
+ "y": 31
+ }
+ },
+ {
+ "kind": "GridLayoutItem",
+ "spec": {
+ "element": {
+ "kind": "ElementReference",
+ "name": "panel-11"
+ },
+ "height": 17,
+ "width": 24,
+ "x": 0,
+ "y": 38
+ }
+ }
+ ]
+ }
+ },
+ "links": [],
+ "liveNow": false,
+ "preload": false,
+ "tags": [
+ "docker",
+ "cadvisor",
+ "node-exporter",
+ "prometheus"
+ ],
+ "timeSettings": {
+ "autoRefresh": "30s",
+ "autoRefreshIntervals": [
"5s",
"10s",
"30s",
@@ -1411,31 +2229,70 @@
"5m",
"15m",
"30m",
- "1h",
- "2h",
- "1d"
+ "1h"
],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
+ "fiscalYearStartMonth": 0,
+ "from": "now-15m",
+ "hideTimepicker": false,
+ "timezone": "browser",
+ "to": "now"
},
- "templating": {
- "list": []
- },
- "annotations": {
- "list": []
- },
- "refresh": "10s",
- "schemaVersion": 12,
- "version": 2,
- "links": [],
- "gnetId": null
+ "title": "Docker Host and Containers",
+ "variables": [
+ {
+ "kind": "DatasourceVariable",
+ "spec": {
+ "allowCustomValue": true,
+ "current": {
+ "text": "prometheus",
+ "value": "bfvc5t62uwd1cf"
+ },
+ "hide": "dontHide",
+ "includeAll": false,
+ "label": "Prometheus",
+ "multi": false,
+ "name": "DS_PROMETHEUS",
+ "options": [],
+ "pluginId": "prometheus",
+ "refresh": "onDashboardLoad",
+ "regex": "",
+ "skipUrlSync": false
+ }
+ },
+ {
+ "kind": "QueryVariable",
+ "spec": {
+ "allValue": ".*",
+ "allowCustomValue": true,
+ "current": {
+ "text": "All",
+ "value": [
+ "$__all"
+ ]
+ },
+ "hide": "dontHide",
+ "includeAll": true,
+ "label": "Container",
+ "multi": true,
+ "name": "container",
+ "options": [],
+ "query": {
+ "datasource": {
+ "name": "${DS_PROMETHEUS}"
+ },
+ "group": "prometheus",
+ "kind": "DataQuery",
+ "spec": {
+ "__legacyStringValue": "label_values(container_last_seen{image!=\"\"}, name)"
+ },
+ "version": "v0"
+ },
+ "refresh": "onTimeRangeChanged",
+ "regex": "",
+ "regexApplyTo": "value",
+ "skipUrlSync": false,
+ "sort": "disabled"
+ }
+ }
+ ]
}
diff --git a/prometheus_grafana_loki/dashboards/monitoring_services.json b/prometheus_grafana_loki/dashboards/monitoring_services.json
deleted file mode 100644
index 00df64a..0000000
--- a/prometheus_grafana_loki/dashboards/monitoring_services.json
+++ /dev/null
@@ -1,3412 +0,0 @@
-{
- "id": null,
- "title": "Monitoring Services",
- "tags": [
- "prometheus"
- ],
- "style": "dark",
- "timezone": "browser",
- "editable": true,
- "hideControls": false,
- "sharedCrosshair": true,
- "panels": [
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "decimals": 1,
- "editable": true,
- "error": false,
- "format": "s",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 0,
- "y": 0
- },
- "hideTimeOverride": true,
- "id": 1,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "s",
- "postfixFontSize": "80%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "tableColumn": "",
- "targets": [
- {
- "expr": "(time() - process_start_time_seconds{instance=\"localhost:9090\",job=\"prometheus\"})",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": "",
- "timeFrom": "10s",
- "timeShift": null,
- "title": "Prometheus Uptime",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg"
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "format": "bytes",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 6,
- "y": 0
- },
- "hideTimeOverride": true,
- "id": 5,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "tableColumn": "",
- "targets": [
- {
- "expr": "sum(container_memory_usage_bytes{container_label_org_label_schema_group=\"monitoring\"})",
- "format": "time_series",
- "interval": "10s",
- "intervalFactor": 1,
- "legendFormat": "",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": "",
- "timeFrom": "10s",
- "timeShift": null,
- "title": "Memory Usage",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg"
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 12,
- "y": 0
- },
- "hideTimeOverride": true,
- "id": 3,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "tableColumn": "",
- "targets": [
- {
- "expr": "prometheus_tsdb_head_chunks",
- "interval": "10s",
- "intervalFactor": 1,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": "",
- "timeFrom": "10s",
- "timeShift": null,
- "title": "In-Memory Chunks",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg"
- },
- {
- "cacheTimeout": null,
- "colorBackground": false,
- "colorValue": false,
- "colors": [
- "rgba(245, 54, 54, 0.9)",
- "rgba(237, 129, 40, 0.89)",
- "rgba(50, 172, 45, 0.97)"
- ],
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 18,
- "y": 0
- },
- "hideTimeOverride": true,
- "id": 2,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false
- },
- "tableColumn": "",
- "targets": [
- {
- "expr": "prometheus_tsdb_head_series",
- "interval": "10s",
- "intervalFactor": 1,
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": "",
- "timeFrom": "10s",
- "timeShift": null,
- "title": "In-Memory Series",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "N/A",
- "value": "null"
- }
- ],
- "valueName": "avg"
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 3
- },
- "id": 6,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (name) (rate(container_cpu_user_seconds_total{container_label_org_label_schema_group=\"monitoring\"}[1m]) * 100 / scalar(count(node_cpu{mode=\"user\"})))",
- "format": "time_series",
- "hide": true,
- "intervalFactor": 10,
- "legendFormat": "{{ name }}",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "sum by (name) (rate(container_cpu_user_seconds_total{container_label_org_label_schema_group=\"monitoring\"}[1m]) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"})))",
- "format": "time_series",
- "intervalFactor": 10,
- "legendFormat": "{{ name }}",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Container CPU Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percent",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "decimals": 2,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 10
- },
- "id": 7,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": false,
- "max": true,
- "min": true,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (name) (container_memory_usage_bytes{container_label_org_label_schema_group=\"monitoring\"})",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 10,
- "legendFormat": "{{ name }}",
- "metric": "container_memory_usage_bytes",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Container Memory Usage",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 2,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "decimals": 0,
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 24,
- "x": 0,
- "y": 17
- },
- "id": 73,
- "legend": {
- "alignAsTable": true,
- "avg": false,
- "current": true,
- "max": false,
- "min": false,
- "rightSide": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": false,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": true,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(ALERTS{alertstate=\"firing\"}) by (alertname)",
- "format": "time_series",
- "interval": "30s",
- "intervalFactor": 1,
- "legendFormat": "{{ alertname }}",
- "metric": "container_memory_usage_bytes",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Alerts",
- "tooltip": {
- "msResolution": true,
- "shared": false,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 22,
- "panels": [],
- "repeat": null,
- "title": "Prometheus Metrics",
- "type": "row"
- },
- {
- "aliasColors": {
- "Max": "#e24d42",
- "Open": "#508642"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 25
- },
- "id": 18,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "process_max_fds{job=\"prometheus\"}",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Max",
- "refId": "A"
- },
- {
- "expr": "process_open_fds{job=\"prometheus\"}",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "Open",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "File Descriptors",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Allocated bytes": "#7EB26D",
- "Allocated bytes - 1m max": "#BF1B00",
- "Allocated bytes - 1m min": "#BF1B00",
- "Allocated bytes - 5m max": "#BF1B00",
- "Allocated bytes - 5m min": "#BF1B00",
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "RSS": "#447EBC"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "decimals": null,
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 25
- },
- "id": 58,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/-/",
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "process_resident_memory_bytes{job=\"prometheus\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "RSS",
- "metric": "process_resident_memory_bytes",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Target heap size",
- "metric": "go_memstats_alloc_bytes",
- "refId": "D",
- "step": 10
- },
- {
- "expr": "go_memstats_next_gc_bytes{job=\"prometheus\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Next GC",
- "metric": "go_memstats_next_gc_bytes",
- "refId": "C",
- "step": 10
- },
- {
- "expr": "go_memstats_alloc_bytes{job=\"prometheus\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Allocated",
- "metric": "go_memstats_alloc_bytes",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Allocated bytes": "#F9BA8F",
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "RSS": "#890F02"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 25
- },
- "id": 60,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\"}[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Allocated",
- "metric": "go_memstats_alloc_bytes",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Allocations",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "Time series": "#70dbed"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 32
- },
- "id": 20,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "prometheus_tsdb_head_series",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Time series",
- "metric": "prometheus_local_storage_memory_series",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Head Time series",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 32
- },
- "id": 24,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "prometheus_tsdb_head_active_appenders",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Head Appenders",
- "metric": "prometheus_local_storage_memory_series",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Head Active Appenders",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "samples/s": "#e5a8e2"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 32
- },
- "id": 26,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_head_samples_appended_total[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Samples",
- "metric": "prometheus_local_storage_ingested_samples_total",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Samples Appended",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": "",
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "To persist": "#9AC48A"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 39
- },
- "id": 28,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/Max.*/",
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "prometheus_tsdb_head_chunks",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Chunks",
- "metric": "prometheus_local_storage_memory_chunks",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Head Chunks",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 39
- },
- "id": 30,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_head_chunks_created_total[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Created",
- "metric": "prometheus_local_storage_chunk_ops_total",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Head Chunks Created",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "Removed": "#e5ac0e"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 39
- },
- "id": 32,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_head_chunks_removed_total[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Removed",
- "metric": "prometheus_local_storage_chunk_ops_total",
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Head Chunks Removed",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max": "#447ebc",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "Min": "#447ebc",
- "Now": "#7eb26d"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 46
- },
- "id": 34,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Max",
- "fillBelowTo": "Min",
- "lines": false
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "prometheus_tsdb_head_min_time",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Min",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "time() * 1000",
- "format": "time_series",
- "hide": false,
- "intervalFactor": 2,
- "legendFormat": "Now",
- "refId": "C"
- },
- {
- "expr": "prometheus_tsdb_head_max_time",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Max",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Head Time Range",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": null,
- "format": "dateTimeAsIso",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 46
- },
- "id": 36,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "GC Time",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Head GC Time/s",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 46
- },
- "id": 38,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Queue length",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "prometheus_tsdb_blocks_loaded",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Blocks Loaded",
- "metric": "prometheus_local_storage_indexing_batch_sizes_sum",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Blocks Loaded",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Failed Compactions": "#bf1b00",
- "Failed Reloads": "#bf1b00",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 53
- },
- "id": 40,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_reloads_total[10m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Reloads",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "TSDB Reloads",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Failed Compactions": "#bf1b00",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 53
- },
- "id": 44,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_wal_corruptions_total[10m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "WAL Corruptions",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "rate(prometheus_tsdb_reloads_failures_total[10m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Reload Failures",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "rate(prometheus_tsdb_head_series_not_found[10m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Head Series Not Found",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "C",
- "step": 10
- },
- {
- "expr": "rate(prometheus_tsdb_compactions_failed_total[10m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Compaction Failures",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "D",
- "step": 10
- },
- {
- "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total[10m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Retention Cutoff Failures",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "E",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "TSDB Problems",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Failed Compactions": "#bf1b00",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 53
- },
- "id": 42,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count[1m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Fsync Latency",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count[1m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Truncate Latency",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "B",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "WAL Latencies",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Failed Compactions": "#bf1b00",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 60
- },
- "id": 46,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_compactions_total[10m])",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Compactions",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Compactions",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 60
- },
- "id": 48,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum[10m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Compaction Time",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Compaction Time",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Allocated bytes": "#F9BA8F",
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833",
- "RSS": "#890F02"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 60
- },
- "id": 50,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_retention_cutoffs_total[10m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Retention Cutoffs",
- "metric": "last",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Retention Cutoffs",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 67
- },
- "id": 56,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count[10m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Chunk Samples",
- "metric": "prometheus_local_storage_series_chunks_persisted_count",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "First Compaction, Avg Chunk Samples",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "none",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 67
- },
- "id": 10,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_target_interval_length_seconds_count[5m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{ interval }}",
- "refId": "A",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Target Scrapes",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "grid": {},
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 67
- },
- "id": 11,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "connected",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{quantile}} ({{interval}})",
- "refId": "A",
- "step": 2
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Scrape Duration",
- "tooltip": {
- "msResolution": true,
- "shared": true,
- "sort": 0,
- "value_type": "cumulative"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "description": "",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 74
- },
- "id": 62,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_http_request_duration_seconds_count[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{handler}}",
- "metric": "prometheus_local_storage_memory_chunkdescs",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "HTTP requests",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "description": "",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 74
- },
- "id": 64,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_http_request_duration_seconds_sum[1m]) / rate(prometheus_http_request_duration_seconds_count[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{handler}}",
- "metric": "prometheus_local_storage_memory_chunkdescs",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "HTTP request latency",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "description": "",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 74
- },
- "id": 66,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_http_request_duration_seconds_sum[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{handler}}",
- "metric": "prometheus_local_storage_memory_chunkdescs",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Time spent in HTTP requests",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "description": "Time spent in each mode, per second",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 81
- },
- "id": 68,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_engine_query_duration_seconds_sum[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "{{slice}}",
- "metric": "prometheus_local_storage_memory_chunkdescs",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Query engine timings",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 81
- },
- "id": 70,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_rule_group_iterations_missed_total[1m]) ",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Rule group missed",
- "metric": "prometheus_local_storage_memory_chunkdescs",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "rate(prometheus_rule_evaluation_failures_total[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Rule evals failed",
- "metric": "prometheus_local_storage_memory_chunkdescs",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rule group evaulation problems",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "aliasColors": {
- "Chunks": "#1F78C1",
- "Chunks to persist": "#508642",
- "Max chunks": "#052B51",
- "Max to persist": "#3F6833"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "Prometheus",
- "editable": true,
- "error": false,
- "fill": 1,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 16,
- "y": 81
- },
- "id": 72,
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 2,
- "links": [],
- "nullPointMode": "null",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(prometheus_rule_group_duration_seconds_sum[1m])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Rule evaluation duration",
- "metric": "prometheus_local_storage_memory_chunkdescs",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeShift": null,
- "title": "Evaluation time of rule groups",
- "tooltip": {
- "msResolution": false,
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- }
- ],
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "templating": {
- "list": []
- },
- "annotations": {
- "list": []
- },
- "refresh": "10s",
- "schemaVersion": 12,
- "version": 22,
- "links": [],
- "gnetId": null
-}
diff --git a/prometheus_grafana_loki/readme.md b/prometheus_grafana_loki/readme.md
index d409421..84be591 100644
--- a/prometheus_grafana_loki/readme.md
+++ b/prometheus_grafana_loki/readme.md
@@ -91,15 +91,22 @@ Prometheus is relatively young project, it is a **pull type** monitoring.
* `prometheus.yml` - a configuration file for prometheus
The three files must be provided.
-The directories are created by docker compose on the first run.
+Usually the directories are created by docker compose on the first run,
+but now we want to create them manualy because of ownership issues.
+Previously we just run the containers as root to avoid dealing with it,
+but since thats not recommended...
+
+be in terminal, in your directory containing the composes file.
+
+* `mkdir -p prometheus_data grafana_data` - create the direcotries
+* `sudo chown 65534:65534 prometheus_data && sudo chown 472:472 grafana_data` - change the ownership
# docker-compose
* **Prometheus** - The official image used. Few extra commands passing configuration.
- Of note is 240 hours(10days) **retention** policy.
+ Of note is 240 hours(10days) or 10GB max **retention** policy.
* **Grafana** - The official image used. Bind mounted directory
- for persistent data storage. User sets **as root**, as it solves issues I am
- lazy to investigate, likely me editing some files as root.
+ for persistent data storage.
* **NodeExporter** - An exporter for linux machines,
in this case gathering the **metrics** of the docker host,
like uptime, cpu load, memory use, network bandwidth use, disk space,...
@@ -107,7 +114,9 @@ The directories are created by docker compose on the first run.
* **cAdvisor** - An exporter for gathering docker **containers** metrics,
showing cpu, memory, network use of **each container**
Runs in `privileged` mode and has some bind mounts of system directories
- to have access to required info.
+ to have access to required info.
+ The three commands - housekeeping, docker only, disable metrics are there
+ to lesser the cpu load of this container.
*Note* - ports are only `expose`, since expectation of use of a reverse proxy
and accessing the services by hostname, not ip and port.
@@ -118,49 +127,50 @@ services:
# MONITORING SYSTEM AND THE METRICS DATABASE
prometheus:
- image: prom/prometheus:v2.42.0
+ image: prom/prometheus:v3.13.2
container_name: prometheus
hostname: prometheus
- user: root
restart: unless-stopped
- depends_on:
- - cadvisor
+ init: true
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--storage.tsdb.retention.time=240h'
+ - '--storage.tsdb.retention.size=10GB'
- '--web.enable-lifecycle'
+ - '--web.enable-admin-api'
volumes:
- ./prometheus_data:/prometheus
- - ./prometheus.yml:/etc/prometheus/prometheus.yml
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
expose:
- "9090"
- labels:
- org.label-schema.group: "monitoring"
# WEB BASED UI VISUALISATION OF METRICS
grafana:
- image: grafana/grafana:9.4.3
+ image: grafana/grafana:13.1.3
container_name: grafana
hostname: grafana
- user: root
restart: unless-stopped
+ init: true
env_file: .env
volumes:
- ./grafana_data:/var/lib/grafana
expose:
- "3000"
- labels:
- org.label-schema.group: "monitoring"
# HOST LINUX MACHINE METRICS EXPORTER
nodeexporter:
- image: prom/node-exporter:v1.5.0
+ image: prom/node-exporter:v1.12.1
container_name: nodeexporter
hostname: nodeexporter
restart: unless-stopped
+ init: true
+ network_mode: host
+ pid: host
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
@@ -172,15 +182,14 @@ services:
- /:/rootfs:ro
expose:
- "9100"
- labels:
- org.label-schema.group: "monitoring"
# DOCKER CONTAINERS METRICS EXPORTER
cadvisor:
- image: gcr.io/cadvisor/cadvisor:v0.47.1
+ image: ghcr.io/google/cadvisor:v0.60.5
container_name: cadvisor
hostname: cadvisor
restart: unless-stopped
+ init: true
privileged: true
devices:
- /dev/kmsg:/dev/kmsg
@@ -189,15 +198,16 @@ services:
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker:/var/lib/docker:ro
- - /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
expose:
- - "3000"
- labels:
- org.label-schema.group: "monitoring"
+ - "8080"
+ command:
+ - --housekeeping_interval=30s
+ - --docker_only=true
+ - --disable_metrics=tcp,udp,percpu,sched,process,hugetlb,referenced_memory,cpu_topology,resctrl
networks:
default:
- name: $DOCKER_MY_NETWORK
+ name: caddy_net
external: true
```
@@ -205,7 +215,6 @@ networks:
```bash
# GENERAL
-DOCKER_MY_NETWORK=caddy_net
TZ=Europe/Bratislava
# GRAFANA
@@ -214,13 +223,12 @@ GF_SECURITY_ADMIN_PASSWORD=admin
GF_USERS_ALLOW_SIGN_UP=false
# GRAFANA EMAIL
GF_SMTP_ENABLED=true
-GF_SMTP_HOST=smtp-relay.sendinblue.com:587
+GF_SMTP_HOST=smtp-relay.brevo.com:587
GF_SMTP_USER=example@gmail.com
GF_SMTP_PASSWORD=xzu0dfFhn3eqa
```
**All containers must be on the same network**.
-Which is named in the `.env` file.
If one does not exist yet: `docker network create caddy_net`
## prometheus.yml
@@ -232,13 +240,13 @@ Contains the bare minimum settings of targets from where metrics are to be pulle
`prometheus.yml`
```yml
global:
- scrape_interval: 15s
- evaluation_interval: 15s
+ scrape_interval: 30s
+ evaluation_interval: 30s
scrape_configs:
- job_name: 'nodeexporter'
static_configs:
- - targets: ['nodeexporter:9100']
+ - targets: ['host.docker.internal:9100']
- job_name: 'cadvisor'
static_configs:
@@ -268,26 +276,18 @@ prom.{$MY_DOMAIN} {
## First run and Grafana configuration
* Login **admin/admin** to `graf.example.com`, change the password.
-* **Add** Prometheus as a **Data source** in Configuration
+* **Add** Prometheus as a **Data source** in Connections
Set **URL** to `http://prometheus:9090`
+ Set **Scrape interval** to `30s`
* **Import** dashboards from [json files in this repo](dashboards/)
-These **dashboards** are the preconfigured ones from
-[stefanprodan/dockprom](https://github.com/stefanprodan/dockprom)
-with **few changes**.
-**Docker host** dashboard did not show free disk space for me, **had to change fstype**
-from `aufs` to `ext4`.
-Also included is [a fix](https://github.com/stefanprodan/dockprom/issues/18#issuecomment-487023049)
-for **host network monitoring** not showing traffick. In all of them
-the default time interval is set to 1h instead of 15m
+**Dashboards** used in the past were the preconfigured ones from
+[stefanprodan/dockprom](https://github.com/stefanprodan/dockprom),
+but with grafana v12 a new dashboard schema and so the old dashboard dont work anymore.
-* **docker_host.json** - dashboard showing linux docker host metrics
-* **docker_containers.json** - dashboard showing docker containers metrics,
- except the ones labeled as `monitoring` in the compose file
-* **monitoring_services.json** - dashboar showing docker containers metrics
- of containers that are labeled `monitoring`
+So I had Claude make some basic dashboard to monitor a docker hosts and the containers.
-
+
---
---
@@ -350,7 +350,6 @@ Links
# Pushgateway
Gives freedom to **push** information in to prometheus from **anywhere**.
-
Be aware that it should **not be abused** to turn prometheus in to push type
monitoring. It is only intented for
[specific situations.](https://github.com/prometheus/pushgateway/blob/master/README.md)
@@ -369,7 +368,7 @@ To **add** pushgateway functionality to the current stack:
# PUSHGATEWAY FOR PROMETHEUS
pushgateway:
- image: prom/pushgateway:v1.5.1
+ image: prom/pushgateway:v1.11.3
container_name: pushgateway
hostname: pushgateway
restart: unless-stopped
@@ -380,7 +379,7 @@ To **add** pushgateway functionality to the current stack:
networks:
default:
- name: $DOCKER_MY_NETWORK
+ name: caddy_net
external: true
```
@@ -409,8 +408,8 @@ To **add** pushgateway functionality to the current stack:
```yml
global:
- scrape_interval: 15s
- evaluation_interval: 15s
+ scrape_interval: 30s
+ evaluation_interval: 30s
scrape_configs:
- job_name: 'pushgateway-scrape'
@@ -446,10 +445,11 @@ To **wipe** the pushgateway clean
### The real world use
* [**Veeam Prometheus Grafana**](https://github.com/DoTheEvo/veeam-prometheus-grafana)
+* [**Kopia Prometheus Grafana**](https://github.com/DoTheEvo/kopia-prometheus-grafana)
-Linked above is a guide-by-example with more info on **pushgateway setup**.
+Linked above are guide-by-examples with more info on **pushgateway setup**.
A real world use to **monitor backups**, along with pushing metrics
-from **windows in powershell**.
+from **windows in powershell** for veeam, or with **golang** for kopia.
@@ -458,196 +458,8 @@ from **windows in powershell**.
# Alertmanager
-To send a **notification** about some **metric** breaching some preset **condition**.
-Notifications channels used here are **email** and
-[**ntfy**](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal)
-
-*Note*
-I myself am **not** planning on using alertmanager.
-Grafana can do alerts for both logs and metrics.
-
-
-
-## The setup
-
-To **add** alertmanager to the current stack:
-
-* **New file** - `alertmanager.yml` to be **bind mounted** in alertmanager container.
- This is the **configuration** on how and where **to deliver** alerts.
- Correct smtp or ntfy info needs to be filled out.
-
-
- alertmanager.yml
-
- ```yml
- route:
- receiver: 'email'
-
- receivers:
- - name: 'ntfy'
- webhook_configs:
- - url: 'https://ntfy.example.com/alertmanager'
- send_resolved: true
-
- - name: 'email'
- email_configs:
- - to: 'whoever@example.com'
- from: 'alertmanager@example.com'
- smarthost: smtp-relay.sendinblue.com:587
- auth_username: ''
- auth_identity: ''
- auth_password: ''
- ```
-
-
-* **New file** - `alert.rules` to be **bind mounted** in to prometheus container
- This file **defines** at what value a metric becomes an **alert** event.
-
-
- alert.rules
-
- ```yml
- groups:
- - name: host
- rules:
- - alert: DiskSpaceLow
- expr: sum(node_filesystem_free_bytes{fstype="ext4"}) > 19
- for: 10s
- labels:
- severity: critical
- annotations:
- description: "Diskspace is low!"
- ```
-
-
-* **Changed** `prometheus.yml`. Added **alerting section** that points to alertmanager
- container, and also **set path** to a `rules` file.
-
-
- prometheus.yml
-
- ```yml
- global:
- scrape_interval: 15s
- evaluation_interval: 15s
-
- scrape_configs:
- - job_name: 'nodeexporter'
- static_configs:
- - targets: ['nodeexporter:9100']
-
- - job_name: 'cadvisor'
- static_configs:
- - targets: ['cadvisor:8080']
-
- - job_name: 'prometheus'
- static_configs:
- - targets: ['localhost:9090']
-
- alerting:
- alertmanagers:
- - scheme: http
- static_configs:
- - targets:
- - 'alertmanager:9093'
-
- rule_files:
- - '/etc/prometheus/rules/alert.rules'
- ```
-
-
-* **New container** - `alertmanager` added to the compose file and **prometheus
- container** has bind mount **rules file** added.
-
-
- docker-compose.yml
-
- ```yml
- services:
-
- # MONITORING SYSTEM AND THE METRICS DATABASE
- prometheus:
- image: prom/prometheus:v2.42.0
- container_name: prometheus
- hostname: prometheus
- user: root
- restart: unless-stopped
- depends_on:
- - cadvisor
- command:
- - '--config.file=/etc/prometheus/prometheus.yml'
- - '--storage.tsdb.path=/prometheus'
- - '--web.console.libraries=/etc/prometheus/console_libraries'
- - '--web.console.templates=/etc/prometheus/consoles'
- - '--storage.tsdb.retention.time=240h'
- - '--web.enable-lifecycle'
- volumes:
- - ./prometheus_data:/prometheus
- - ./prometheus.yml:/etc/prometheus/prometheus.yml
- - ./alert.rules:/etc/prometheus/rules/alert.rules
- expose:
- - "9090"
- labels:
- org.label-schema.group: "monitoring"
-
- # ALERT MANAGMENT FOR PROMETHEUS
- alertmanager:
- image: prom/alertmanager:v0.25.0
- container_name: alertmanager
- hostname: alertmanager
- user: root
- restart: unless-stopped
- volumes:
- - ./alertmanager.yml:/etc/alertmanager.yml
- - ./alertmanager_data:/alertmanager
- command:
- - '--config.file=/etc/alertmanager.yml'
- - '--storage.path=/alertmanager'
- expose:
- - "9093"
- labels:
- org.label-schema.group: "monitoring"
-
- networks:
- default:
- name: $DOCKER_MY_NETWORK
- external: true
- ```
-
-
-* **Adding** alertmanager to the **Caddyfile** of the reverse proxy so that
- it can be reached at `https://alert.example.com`. **Not necessary**,
- but useful as it **allows to send alerts from anywhere**,
- not just from prometheus, or other containers on same docker network.
-
-
- Caddyfile
-
- ```php
- alert.{$MY_DOMAIN} {
- reverse_proxy alertmanager:9093
- }
- ```
-
-
-## The basics
-
-
-
-
-Once above setup is done, **an alert** about low disk space **should fire**
-and a **notification** email should come.
-In `alertmanager.yml` a switch from email **to ntfy** can be done.
-
-*Useful*
-
-* **alert** from anywhere using **curl**:
- `curl -H 'Content-Type: application/json' -d '[{"labels":{"alertname":"blabla"}}]' https://alert.example.com/api/v1/alerts`
-* **reload rules**:
- `curl -X POST https://prom.example.com/-/reload`
-
-[stefanprodan/dockprom](https://github.com/stefanprodan/dockprom#define-alerts)
-has more detailed section on alerting worth checking out.
+scrapping this section, september 2026
+grafana alerting will be used, probably that one should have a section
---
---
@@ -658,7 +470,6 @@ has more detailed section on alerting worth checking out.
Loki is a log aggregation tool, made by the grafana team.
Sometimes called a Prometheus for logs, it's a **push** type monitoring.
-
It uses [LogQL](https://promcon.io/2019-munich/slides/lt1-08_logql-in-5-minutes.pdf)
for queries, which is similar to PromQL in its use of labels.
@@ -671,12 +482,13 @@ There are two ways to **push logs** to Loki from a docker container.
`/etc/docker/daemon.json` or per container in compose files.
It's the simpler, easier way, but **lacks fine control** over the logs
being pushed.
- * **[Promtail](https://grafana.com/docs/loki/latest/clients/promtail/)**
- deployed as an another **container**, with bind mount of logs it should scrape,
+ * **[Alloy](https://grafana.com/docs/loki/latest/send-data/alloy/)**
+ (successsor of promtail)deployed as an another **container**, with bind mount of logs it should scrape,
and bind mount of its config file. This config file is very powerful,
- giving a lot of **control** how logs are processed and pushed.
+ giving a lot of **control** how logs are processed and pushed.
+ Can also just run with an open port, waiting for others to push logs to it.
-
+
## Loki setup
@@ -692,7 +504,7 @@ There are two ways to **push logs** to Loki from a docker container.
# LOG MANAGMENT WITH LOKI
loki:
- image: grafana/loki:main-0295fd4
+ image: grafana/loki:3.7.6
container_name: loki
hostname: loki
user: root
@@ -704,12 +516,10 @@ There are two ways to **push logs** to Loki from a docker container.
- '-config.file=/etc/loki-config.yml'
ports:
- "3100:3100"
- labels:
- org.label-schema.group: "monitoring"
networks:
default:
- name: $DOCKER_MY_NETWORK
+ name: caddy_net
external: true
```
@@ -745,39 +555,31 @@ There are two ways to **push logs** to Loki from a docker container.
kvstore:
store: inmemory
- # --- disable splitting to fix "too many outstanding requests"
-
query_range:
parallelise_shardable_queries: false
- # --- compactor to have control over length of data retention
-
compactor:
working_directory: /loki/compactor
compaction_interval: 10m
retention_enabled: true
+ delete_request_store: filesystem
retention_delete_delay: 2h
retention_delete_worker_count: 150
limits_config:
- retention_period: 240h
- split_queries_by_interval: 0 # part of disable splitting fix
-
- # -------------------------------------------------------
+ retention_period: 720h
+ split_queries_by_interval: 0
schema_config:
configs:
- - from: 2020-10-24
- store: boltdb-shipper
+ - from: 2026-01-01
+ store: tsdb
object_store: filesystem
- schema: v11
+ schema: v13
index:
prefix: index_
period: 24h
- ruler:
- alertmanager_url: http://alertmanager:9093
-
analytics:
reporting_enabled: false
```
@@ -808,9 +610,9 @@ There are two ways to **push logs** to Loki from a docker container.
```
-* #### promtail
+* #### Alloy
- * Containers that should be monitored with **promtail** need it **added**
+ * Containers that should be monitored with **Alloy** need it **added**
**to** their **compose** file, and made sure that it has access to the log files.
@@ -828,21 +630,22 @@ There are two ways to **push logs** to Loki from a docker container.
tty: true
stdin_open: true
ports:
- - 25565:25565 # minecraft server players connect
+ - 25565:25565
volumes:
- ./minecraft_data:/data
# LOG AGENT PUSHING LOGS TO LOKI
- promtail:
- image: grafana/promtail
- container_name: minecraft-promtail
- hostname: minecraft-promtail
+ alloy:
+ image: grafana/alloy:latest
+ container_name: minecraft-alloy
+ hostname: minecraft-alloy
restart: unless-stopped
volumes:
- ./minecraft_data/logs:/var/log/minecraft:ro
- - ./promtail-config.yml:/etc/promtail-config.yml
+ - ./config.alloy:/etc/alloy/config.alloy:ro
command:
- - '-config.file=/etc/promtail-config.yml'
+ - run
+ - /etc/alloy/config.alloy
networks:
default:
@@ -874,42 +677,51 @@ There are two ways to **push logs** to Loki from a docker container.
- ./caddy_logs:/var/log/caddy
# LOG AGENT PUSHING LOGS TO LOKI
- promtail:
- image: grafana/promtail
- container_name: caddy-promtail
- hostname: caddy-promtail
+ alloy:
+ image: grafana/alloy:latest
+ container_name: caddy-alloy
+ hostname: caddy-alloy
restart: unless-stopped
volumes:
- ./caddy_logs:/var/log/caddy:ro
- - ./promtail-config.yml:/etc/promtail-config.yml
+ - ./config.alloy:/etc/alloy/config.alloy:ro
command:
- - '-config.file=/etc/promtail-config.yml'
+ - run
+ - /etc/alloy/config.alloy
networks:
default:
name: $DOCKER_MY_NETWORK
external: true
-
```
- * Generic **config file for promtail**, needs to be bind mounted
+ * Generic **config file for alloy**, needs to be bind mounted
+ in to alloy container
- promtail-config.yml
+ config.alloy
```yml
- clients:
- - url: http://loki:3100/loki/api/v1/push
+ local.file_match "logs" {
+ path_targets = [
+ {
+ "__path__" = "/var/log/blablabla/*.log",
+ "job" = "blablabla_log",
+ },
+ ]
+ }
- scrape_configs:
- - job_name: blablabla
- static_configs:
- - targets:
- - localhost
- labels:
- job: blablabla_log
- __path__: /var/log/blablabla/*.log
+ loki.source.file "logs" {
+ targets = local.file_match.logs.targets
+ forward_to = [loki.write.default.receiver]
+ }
+
+ loki.write "default" {
+ endpoint {
+ url = "http://loki:3100/loki/api/v1/push"
+ }
+ }
```
@@ -918,13 +730,17 @@ There are two ways to **push logs** to Loki from a docker container.
* In **grafana**, loki needs to be added as a **datasource**, `http://loki:3100`
* In **Explore section**, switch to Loki as source
* if loki-docker-driver then filter by `container_name` or `compose_project`
- * if promtail then filter by job name set in promtail config
- in the labels section
+ * if alloy then filter by job name set in `config.alloy`
If all was set correctly logs should be visible in Grafana.
+
+**work in progress switching from promtail to alloy**
+**work in progress switching from promtail to alloy**
+**work in progress switching from promtail to alloy**
+
# Minecraft Loki example
What can be seen in this example:
@@ -1663,3 +1479,7 @@ that makes daily snapshot of the entire directory.
* delete the entire monitoring directory
* from the backup copy back the monitoring directory
* start the containers `docker-compose up -d`
+
+# Other resources
+
+* [Prometheus: How We Slashed Memory Usage](https://devoriales.com/post/384/prometheus-how-we-slashed-memory-usage)