diff --git a/add-ons/kube-prometheus-stack/dashboard-application-logs.yaml b/add-ons/kube-prometheus-stack/dashboard-application-logs.yaml deleted file mode 100644 index 532bac3..0000000 --- a/add-ons/kube-prometheus-stack/dashboard-application-logs.yaml +++ /dev/null @@ -1,1040 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - grafana_dashboard: "1" - name: grafana-dashboard-application-logs - namespace: monitoring -data: - application-logs.json: | - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": 16966, - "graphTooltip": 0, - "id": null, - "iteration": 1594222083341, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorPostfix": true, - "colorValue": true, - "colors": [ - "rgb(31, 255, 7)", - "rgb(31, 255, 7)", - "rgb(31, 255, 7)" - ], - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "Total Count of log lines in the specified time range", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 11, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "$$hashKey": "object:65", - "name": "value to text", - "value": 1 - }, - { - "$$hashKey": "object:66", - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "pluginVersion": "6.4.3", - "postfix": " lines", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgb(0, 82, 0)", - "full": false, - "lineColor": "rgb(31, 255, 7)", - "show": true, - "ymax": null, - "ymin": null - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", workload=~\"$workload\"})[$__interval]))", - "hide": false, - "refId": "A" - } - ], - "thresholds": "10,50", - "timeFrom": null, - "timeShift": null, - "title": "Total Count of logs", - "type": "singlestat", - "valueFontSize": "70%", - "valueMaps": [ - { - "$$hashKey": "object:68", - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "total" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorPostfix": true, - "colorPrefix": false, - "colorValue": true, - "colors": [ - "rgb(222, 15, 43)", - "rgb(222, 15, 43)", - "rgb(222, 15, 43)" - ], - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "Total Count: of $searchable_pattern in the specified time range", - "fieldConfig": { - "defaults": { - "custom": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 6, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "$$hashKey": "object:1630", - "name": "value to text", - "value": 1 - }, - { - "$$hashKey": "object:1631", - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "pluginVersion": "6.4.3", - "postfix": " \"$searchable_pattern\"", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgb(105, 34, 43)", - "full": false, - "lineColor": "#C4162A", - "show": true, - "ymax": null, - "ymin": null - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", workload=~\"$workload\"} |~ \"(?i)$searchable_pattern\")[$__interval]))", - "hide": false, - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Count: of $searchable_pattern", - "type": "singlestat", - "valueFontSize": "70%", - "valueMaps": [ - { - "$$hashKey": "object:1633", - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "total" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "Live logs is a like 'tail -f' in a real time", - "fieldConfig": { - "defaults": { - "custom": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 2, - "options": { - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "expr": "{namespace=\"$namespace\", workload=~\"$workload\", stream=~\"$stream\"} |~ \"(?i)$searchable_pattern\" ", - "hide": false, - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Live logs", - "type": "logs" - }, - { - "aliasColors": { - "stderr": "#C4162A", - "stdout": "#37872D" - }, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "decimals": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "color": "#299c46", - "text": "0" - } - }, - "type": "special" - } - ], - "unit": "none" - }, - "overrides": [] - }, - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 7, - "w": 9, - "x": 0, - "y": 12 - }, - "id": 19, - "interval": null, - "legend": { - "header": "Total Count", - "percentage": true, - "percentageDecimals": 2, - "show": true, - "sort": "total", - "sortDesc": true, - "values": true - }, - "legendType": "right", - "links": [], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single" - } - }, - "pieType": "pie", - "pluginVersion": "7.0.4", - "strokeWidth": "1.5", - "targets": [ - { - "expr": "sum(count_over_time(({namespace=\"$namespace\", workload=~\"$workload\"})[$__interval])) by (stream)", - "hide": false, - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total count of stderr / stdout pie", - "type": "piechart", - "valueName": "total" - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "decimals": 0, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 7, - "w": 9, - "x": 9, - "y": 12 - }, - "id": 20, - "interval": "1m", - "legend": { - "header": "total count", - "percentage": true, - "show": true, - "sort": "total", - "sortDesc": true, - "values": true - }, - "legendType": "right", - "links": [], - "maxDataPoints": "", - "nullPointMode": "connected", - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single" - } - }, - "pieType": "donut", - "pluginVersion": "6.4.3", - "strokeWidth": "0.4", - "targets": [ - { - "expr": "sum(count_over_time(({namespace=\"$namespace\", workload=~\"$workload\", stream=~\"$stream\"} |~ \"(?i)$searchable_pattern\")[$__interval])) by (pod)", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Matched word: \"$searchable_pattern\" donut", - "type": "piechart", - "valueName": "total" - }, - { - "cacheTimeout": null, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "0", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "noValue": "0", - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 12 - }, - "id": 9, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false - }, - "pluginVersion": "7.0.4", - "targets": [ - { - "expr": "sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", workload=~\"$workload\"} |~ \"(?i)$searchable_pattern\")[$__interval])) * 100 / sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", workload=~\"$workload\"})[$__interval]))", - "hide": false, - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\"$searchable_pattern\" Percentage for specified time", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": true, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 18, - "interval": "1m", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 3, - "links": [], - "maxDataPoints": "", - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.4.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(count_over_time(({namespace=\"$namespace\", workload=~\"$workload\", stream=~\"$stream\"} |~ \"(?i)$searchable_pattern\")[$__interval])) by (pod)", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Matched word: \"$searchable_pattern\" historical", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:425", - "decimals": 0, - "format": "short", - "label": "Count", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:426", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 10, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 10, - "interval": null, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxDataPoints": 100, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.4.3", - "pointradius": 0.5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(({namespace=\"$namespace\", stream=~\"$stream\", workload=~\"$workload\"} |~ \"(?i)$searchable_pattern\")[30s])) by (pod)", - "hide": false, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "\"$searchable_pattern\" Rate per Pod", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:2804", - "format": "short", - "label": null, - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:2805", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "0", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10 - }, - { - "color": "#C4162A", - "value": 50 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "fill": 2, - "fillGradient": 4, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 30 - }, - "hiddenSeries": false, - "id": 7, - "interval": null, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxDataPoints": 100, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "7.0.4", - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:154", - "alias": "{stream=\"stderr\"} stderr", - "color": "#C4162A", - "legend": false, - "linewidth": 2 - }, - { - "$$hashKey": "object:220", - "alias": "{stream=\"stdout\"} stdout", - "color": "#56A64B", - "legend": false, - "linewidth": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(count_over_time(({namespace=\"$namespace\", workload=~\"$workload\"})[$__interval])) by (stream)", - "hide": false, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Count of stderr / stdout historical", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:188", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:189", - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "1m", - "schemaVersion": 25, - "style": "dark", - "tags": [ - "Loki", - "logging" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": true, - "text": "loki", - "value": "loki" - }, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "definition": "label_values({namespace=~\".+\"}, namespace)", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "label_values({namespace=~\".+\"}, namespace)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "definition": "label_values({namespace=\"$namespace\"}, workload)", - "hide": 0, - "includeAll": true, - "label": "Workload", - "multi": false, - "name": "workload", - "options": [], - "query": "label_values({namespace=\"$namespace\"}, workload)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "definition": "label_values({namespace=\"$namespace\"}, stream)", - "hide": 0, - "includeAll": true, - "label": "Stream", - "multi": false, - "name": "stream", - "options": [], - "query": "label_values({namespace=\"$namespace\"}, stream)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": true, - "text": "", - "value": "" - }, - "hide": 0, - "label": "Search (case insensitive)", - "name": "searchable_pattern", - "options": [ - { - "selected": true, - "text": "", - "value": "" - } - ], - "query": "", - "skipUrlSync": false, - "type": "textbox" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Application logs", - "uid": "application-logs", - "version": 2, - "description": "Container/Pod Log Dashboard(Loki as DataSource)" - } diff --git a/add-ons/kube-prometheus-stack/dashboard-http-requests.yaml b/add-ons/kube-prometheus-stack/dashboard-http-requests.yaml deleted file mode 100644 index 621d4fd..0000000 --- a/add-ons/kube-prometheus-stack/dashboard-http-requests.yaml +++ /dev/null @@ -1,213 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - grafana_dashboard: "1" - name: grafana-dashboard-http-requests - namespace: monitoring -data: - http-requests.json: | - { - "uid": "http-requests", - "title": "HTTP requests", - "editable": true, - "schemaVersion": 41, - "refresh": "", - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "tags": [], - "templating": { - "list": [] - }, - "panels": [ - { - "type": "timeseries", - "title": "Requests by host", - "id": 1, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum by (RequestHost) (count_over_time({namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" [$__auto]))", - "legendFormat": "{{RequestHost}}", - "queryType": "range" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "fillOpacity": 80, - "showPoints": "never", - "stacking": { - "mode": "normal" - }, - "drawStyle": "bars" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Responses by status", - "id": 2, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum by (DownstreamStatus) (count_over_time({namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" [$__auto]))", - "legendFormat": "{{DownstreamStatus}}", - "queryType": "range" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "fillOpacity": 80, - "showPoints": "never", - "stacking": { - "mode": "normal" - }, - "drawStyle": "bars" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "p95 response time by host", - "id": 3, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "quantile_over_time(0.95, {namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" | unwrap Duration [$__auto]) by (RequestHost)", - "legendFormat": "{{RequestHost}}", - "queryType": "range" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "ns", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "logs", - "title": "Access log", - "id": 4, - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 16 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "queryType": "range", - "expr": "{namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" | line_format \"{{.DownstreamStatus}} {{.RequestMethod}} {{.RequestHost}}{{.RequestPath}} {{div .Duration 1000000}}ms {{.ClientHost}}\"" - } - ], - "options": { - "showTime": true, - "wrapLogMessage": false, - "sortOrder": "Descending", - "enableLogDetails": true, - "dedupStrategy": "none", - "showLabels": false - } - } - ], - "annotations": { - "list": [] - } - } diff --git a/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml b/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml deleted file mode 100644 index d1c4dee..0000000 --- a/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml +++ /dev/null @@ -1,985 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - grafana_dashboard: "1" - name: grafana-dashboard-raspberry-pi - namespace: monitoring -data: - raspberry-pi.json: | - { - "uid": "raspberry-pi", - "title": "Raspberry Pi", - "editable": true, - "schemaVersion": 41, - "refresh": "1m", - "time": { - "from": "now-6h", - "to": "now" - }, - "timezone": "browser", - "tags": [], - "templating": { - "list": [ - { - "name": "namespace", - "label": "Namespace", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "query": { - "query": "label_values(container_cpu_usage_seconds_total, namespace)", - "refId": "ns" - }, - "definition": "label_values(container_cpu_usage_seconds_total, namespace)", - "multi": true, - "includeAll": true, - "current": { - "text": "All", - "value": "$__all" - }, - "refresh": 2, - "sort": 1 - } - ] - }, - "panels": [ - { - "type": "row", - "title": "Node", - "id": 1, - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "panels": [] - }, - { - "type": "stat", - "title": "CPU", - "id": 2, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", - "instant": true - } - ], - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "decimals": 0 - }, - "overrides": [] - } - }, - { - "type": "stat", - "title": "Memory", - "id": 3, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", - "instant": true - } - ], - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "decimals": 0 - }, - "overrides": [] - } - }, - { - "type": "stat", - "title": "Disk /", - "id": 4, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", - "instant": true - } - ], - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "decimals": 0 - }, - "overrides": [] - } - }, - { - "type": "stat", - "title": "CPU temperature", - "id": 5, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "node_thermal_zone_temp{type=\"cpu-thermal\"}", - "instant": true - } - ], - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "decimals": 0 - }, - "overrides": [] - } - }, - { - "type": "stat", - "title": "NVMe temperature", - "id": 6, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "max(node_hwmon_temp_celsius{chip=\"nvme_nvme0\"})", - "instant": true - } - ], - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 60 - }, - { - "color": "red", - "value": 70 - } - ] - }, - "decimals": 0 - }, - "overrides": [] - } - }, - { - "type": "stat", - "title": "Uptime", - "id": 7, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "time() - node_boot_time_seconds", - "instant": true - } - ], - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "value", - "graphMode": "none" - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "decimals": 0 - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "CPU", - "id": 8, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 5 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", - "legendFormat": "used" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "100 * avg(rate(node_cpu_seconds_total{mode=\"iowait\"}[$__rate_interval]))", - "legendFormat": "iowait" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Memory", - "id": 9, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 5 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", - "legendFormat": "used" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "node_memory_MemAvailable_bytes", - "legendFormat": "available" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "fillOpacity": 10, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Disk I/O", - "id": 10, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum(rate(node_disk_read_bytes_total{device=~\"nvme.+n1|mmcblk0\"}[$__rate_interval]))", - "legendFormat": "read" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum(rate(node_disk_written_bytes_total{device=~\"nvme.+n1|mmcblk0\"}[$__rate_interval]))", - "legendFormat": "written" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "Bps", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Network", - "id": 11, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "rate(node_network_receive_bytes_total{device=~\"eth0|wlan0\"}[$__rate_interval])", - "legendFormat": "{{device}} in" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "rate(node_network_transmit_bytes_total{device=~\"eth0|wlan0\"}[$__rate_interval])", - "legendFormat": "{{device}} out" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "Bps", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Temperatures", - "id": 12, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 21 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "node_thermal_zone_temp{type=\"cpu-thermal\"}", - "legendFormat": "cpu" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "max(node_hwmon_temp_celsius{chip=\"nvme_nvme0\"})", - "legendFormat": "nvme" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "title": "Pods", - "id": 13, - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "panels": [] - }, - { - "type": "timeseries", - "title": "CPU by namespace", - "id": 14, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval]))", - "legendFormat": "{{namespace}}" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "cores", - "custom": { - "fillOpacity": 10, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Memory by namespace", - "id": 15, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=\"\",namespace=~\"$namespace\"})", - "legendFormat": "{{namespace}}" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "fillOpacity": 10, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Top pods by CPU", - "id": 16, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 37 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "topk(10, sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval])))", - "legendFormat": "{{namespace}}/{{pod}}" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "cores", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Top pods by memory", - "id": 17, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 37 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "topk(10, sum by (namespace, pod) (container_memory_working_set_bytes{container!=\"\",namespace=~\"$namespace\"}))", - "legendFormat": "{{namespace}}/{{pod}}" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Top pods by disk writes", - "id": 18, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 45 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "topk(10, sum by (namespace, pod) (rate(container_fs_writes_bytes_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval])))", - "legendFormat": "{{namespace}}/{{pod}}" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "Bps", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "title": "Network by namespace", - "id": 19, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 45 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", - "legendFormat": "{{namespace}} in" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "-sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", - "legendFormat": "{{namespace}} out" - } - ], - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "fieldConfig": { - "defaults": { - "unit": "Bps", - "custom": { - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - } - ], - "annotations": { - "list": [] - } - } diff --git a/add-ons/kube-prometheus-stack/kustomization.yaml b/add-ons/kube-prometheus-stack/kustomization.yaml index 85f302e..6ee7174 100644 --- a/add-ons/kube-prometheus-stack/kustomization.yaml +++ b/add-ons/kube-prometheus-stack/kustomization.yaml @@ -1,9 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - dashboard-application-logs.yaml - - dashboard-http-requests.yaml - - dashboard-raspberry-pi.yaml - grafana-admin-sealedsecret.yaml - grafana-db-sealedsecret.yaml - grafana-oidc-sealedsecret.yaml diff --git a/argocd-apps/add-ons/alloy.yaml b/argocd-apps/add-ons/alloy.yaml index e54f232..9804c40 100644 --- a/argocd-apps/add-ons/alloy.yaml +++ b/argocd-apps/add-ons/alloy.yaml @@ -20,6 +20,9 @@ spec: content: | discovery.kubernetes "pods" { role = "pod" + namespaces { + names = ["garage", "postgres", "redoost", "traefik"] + } selectors { role = "pod" field = "spec.nodeName=" + sys.env("HOSTNAME") diff --git a/argocd-apps/add-ons/kube-prometheus-stack.yaml b/argocd-apps/add-ons/kube-prometheus-stack.yaml index 5a56db0..b6032a9 100644 --- a/argocd-apps/add-ons/kube-prometheus-stack.yaml +++ b/argocd-apps/add-ons/kube-prometheus-stack.yaml @@ -28,10 +28,7 @@ spec: cAdvisorMetricRelabelings: - action: keep sourceLabels: [__name__] - regex: container_(cpu_usage_seconds_total|memory_working_set_bytes|fs_(reads|writes)_bytes_total|network_(receive|transmit)_bytes_total) - - action: drop - sourceLabels: [__name__, interface] - regex: container_network_.+;(br-|cni|docker|flannel|lo|tailscale|tun|veth|wg|wlan).* + regex: container_(cpu_usage_seconds_total|memory_working_set_bytes) - action: drop sourceLabels: [id, pod] regex: .+; @@ -71,12 +68,26 @@ spec: serviceMonitor: selfMonitor: false prometheus-node-exporter: - # The first two are the chart's defaults, as lists are replaced. + # CPU, memory, disks, network and temperature. extraArgs: + - --collector.disable-defaults + - --collector.cpu + - --collector.meminfo + - --collector.filesystem + - --collector.netdev + - --collector.hwmon + - --web.disable-exporter-metrics + # The chart's defaults, as lists are replaced. - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/) - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$ - - --collector.netclass.ignored-devices=^(veth.*|br-.*|docker0|cni0|lo)$ - --collector.netdev.device-exclude=^(veth.*|br-.*|docker0|cni0|lo)$ + prometheus: + monitor: + # Collectors report more than these, like per-chip sensor labels. + metricRelabelings: + - action: keep + sourceLabels: [__name__] + regex: node_(cpu_seconds_total|memory_(MemAvailable|MemTotal)_bytes|filesystem_(avail|size)_bytes|network_(receive|transmit)_bytes_total|hwmon_temp_celsius) resources: limits: cpu: 200m