From 27dc30693b0e5126e59277ecebbd928c8814d9c6 Mon Sep 17 00:00:00 2001 From: Karan Parashar Date: Sat, 10 Oct 2026 12:45:21 +0530 Subject: [PATCH] feat(monitoring): Scrape node and pod metrics, add logs and Pi dashboards --- add-ons/authentik/blueprint-argocd.yaml | 1 - add-ons/authentik/blueprint-outpost.yaml | 1 - .../authentik/blueprint-wireguard-admin.yaml | 5 +- .../dashboard-application-logs.yaml | 1040 +++++++++++++++++ .../dashboard-http-requests.yaml | 213 ++++ .../dashboard-raspberry-pi.yaml | 985 ++++++++++++++++ .../kube-prometheus-stack/kustomization.yaml | 4 +- .../kube-prometheus-stack/scrapeconfigs.yaml | 59 - add-ons/postgres/cluster.yaml | 4 +- apps/pgweb/deployment.yaml | 2 - apps/pgweb/networkpolicy.yaml | 3 +- apps/wireguard/deployment.yaml | 12 +- .../add-ons/kube-prometheus-stack.yaml | 72 +- argocd-apps/add-ons/traefik.yaml | 7 + 14 files changed, 2298 insertions(+), 110 deletions(-) create mode 100644 add-ons/kube-prometheus-stack/dashboard-application-logs.yaml create mode 100644 add-ons/kube-prometheus-stack/dashboard-http-requests.yaml create mode 100644 add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml delete mode 100644 add-ons/kube-prometheus-stack/scrapeconfigs.yaml diff --git a/add-ons/authentik/blueprint-argocd.yaml b/add-ons/authentik/blueprint-argocd.yaml index ffe7ed1..041fbf3 100644 --- a/add-ons/authentik/blueprint-argocd.yaml +++ b/add-ons/authentik/blueprint-argocd.yaml @@ -9,7 +9,6 @@ data: metadata: name: ArgoCD entries: - # Public client: the CLI can't keep a secret, so ArgoCD uses PKCE instead. - model: authentik_providers_oauth2.oauth2provider id: argocd-provider identifiers: diff --git a/add-ons/authentik/blueprint-outpost.yaml b/add-ons/authentik/blueprint-outpost.yaml index 0ddc35b..5cb6913 100644 --- a/add-ons/authentik/blueprint-outpost.yaml +++ b/add-ons/authentik/blueprint-outpost.yaml @@ -24,7 +24,6 @@ data: identifiers: path: mounted/cm-blueprint-qbittorrent/qbittorrent.yaml required: true - # Serves forward auth for the apps behind the authentik middleware. - model: authentik_outposts.outpost identifiers: managed: goauthentik.io/outposts/embedded diff --git a/add-ons/authentik/blueprint-wireguard-admin.yaml b/add-ons/authentik/blueprint-wireguard-admin.yaml index acbab2b..0ca26b6 100644 --- a/add-ons/authentik/blueprint-wireguard-admin.yaml +++ b/add-ons/authentik/blueprint-wireguard-admin.yaml @@ -9,8 +9,7 @@ data: metadata: name: WireGuard admin entries: - # wg-easy rejects logins unless email_verified is true; the default email - # mapping always sends false. + # wg-easy requires email_verified, which the default mapping sends as false. - model: authentik_providers_oauth2.scopemapping id: verified-email identifiers: @@ -52,7 +51,7 @@ data: name: WireGuard Admin provider: !KeyOf wireguard-admin-provider meta_launch_url: https://wg-admin.whyredfire.dev - # wg-easy makes every user it registers an admin, so only admins may log in. + # wg-easy makes every user an admin. - model: authentik_policies.policybinding identifiers: order: 0 diff --git a/add-ons/kube-prometheus-stack/dashboard-application-logs.yaml b/add-ons/kube-prometheus-stack/dashboard-application-logs.yaml new file mode 100644 index 0000000..772ca9e --- /dev/null +++ b/add-ons/kube-prometheus-stack/dashboard-application-logs.yaml @@ -0,0 +1,1040 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: grafana-dashboard-application-logs + namespace: monitoring +data: + application-logs.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 16966, + "graphTooltip": 0, + "id": null, + "iteration": 1594222083341, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorPostfix": true, + "colorValue": true, + "colors": [ + "rgb(31, 255, 7)", + "rgb(31, 255, 7)", + "rgb(31, 255, 7)" + ], + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Total Count of log lines in the specified time range", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "$$hashKey": "object:65", + "name": "value to text", + "value": 1 + }, + { + "$$hashKey": "object:66", + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.4.3", + "postfix": " lines", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgb(0, 82, 0)", + "full": false, + "lineColor": "rgb(31, 255, 7)", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", pod=~\"$pod\"})[$__interval]))", + "hide": false, + "refId": "A" + } + ], + "thresholds": "10,50", + "timeFrom": null, + "timeShift": null, + "title": "Total Count of logs", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "$$hashKey": "object:68", + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPostfix": true, + "colorPrefix": false, + "colorValue": true, + "colors": [ + "rgb(222, 15, 43)", + "rgb(222, 15, 43)", + "rgb(222, 15, 43)" + ], + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Total Count: of $searchable_pattern in the specified time range", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "$$hashKey": "object:1630", + "name": "value to text", + "value": 1 + }, + { + "$$hashKey": "object:1631", + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.4.3", + "postfix": " \"$searchable_pattern\"", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgb(105, 34, 43)", + "full": false, + "lineColor": "#C4162A", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", pod=~\"$pod\"} |~ \"(?i)$searchable_pattern\")[$__interval]))", + "hide": false, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Count: of $searchable_pattern", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "$$hashKey": "object:1633", + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "total" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Live logs is a like 'tail -f' in a real time", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "expr": "{namespace=\"$namespace\", pod=~\"$pod\", stream=~\"$stream\"} |~ \"(?i)$searchable_pattern\" ", + "hide": false, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Live logs", + "type": "logs" + }, + { + "aliasColors": { + "stderr": "#C4162A", + "stdout": "#37872D" + }, + "breakPoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "decimals": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "color": "#299c46", + "text": "0" + } + }, + "type": "special" + } + ], + "unit": "none" + }, + "overrides": [] + }, + "fontSize": "80%", + "format": "short", + "gridPos": { + "h": 7, + "w": 9, + "x": 0, + "y": 12 + }, + "id": 19, + "interval": null, + "legend": { + "header": "Total Count", + "percentage": true, + "percentageDecimals": 2, + "show": true, + "sort": "total", + "sortDesc": true, + "values": true + }, + "legendType": "right", + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "pieType": "pie", + "pluginVersion": "7.0.4", + "strokeWidth": "1.5", + "targets": [ + { + "expr": "sum(count_over_time(({namespace=\"$namespace\", pod=~\"$pod\"})[$__interval])) by (stream)", + "hide": false, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total count of stderr / stdout pie", + "type": "piechart", + "valueName": "total" + }, + { + "aliasColors": {}, + "breakPoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "fontSize": "80%", + "format": "short", + "gridPos": { + "h": 7, + "w": 9, + "x": 9, + "y": 12 + }, + "id": 20, + "interval": "1m", + "legend": { + "header": "total count", + "percentage": true, + "show": true, + "sort": "total", + "sortDesc": true, + "values": true + }, + "legendType": "right", + "links": [], + "maxDataPoints": "", + "nullPointMode": "connected", + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "pieType": "donut", + "pluginVersion": "6.4.3", + "strokeWidth": "0.4", + "targets": [ + { + "expr": "sum(count_over_time(({namespace=\"$namespace\", pod=~\"$pod\", stream=~\"$stream\"} |~ \"(?i)$searchable_pattern\")[$__interval])) by (pod)", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Matched word: \"$searchable_pattern\" donut", + "type": "piechart", + "valueName": "total" + }, + { + "cacheTimeout": null, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "0", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "noValue": "0", + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 9, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": false + }, + "pluginVersion": "7.0.4", + "targets": [ + { + "expr": "sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", pod=~\"$pod\"} |~ \"(?i)$searchable_pattern\")[$__interval])) * 100 / sum(count_over_time(({namespace=\"$namespace\", stream=~\"$stream\", pod=~\"$pod\"})[$__interval]))", + "hide": false, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "\"$searchable_pattern\" Percentage for specified time", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": true, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 3, + "links": [], + "maxDataPoints": "", + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.4.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(count_over_time(({namespace=\"$namespace\", pod=~\"$pod\", stream=~\"$stream\"} |~ \"(?i)$searchable_pattern\")[$__interval])) by (pod)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Matched word: \"$searchable_pattern\" historical", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:425", + "decimals": 0, + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:426", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 10, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 10, + "interval": null, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.4.3", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(({namespace=\"$namespace\", stream=~\"$stream\", pod=~\"$pod\"} |~ \"(?i)$searchable_pattern\")[30s])) by (pod)", + "hide": false, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "\"$searchable_pattern\" Rate per Pod", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2804", + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2805", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "0", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "#C4162A", + "value": 50 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 4, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 7, + "interval": null, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "7.0.4", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:154", + "alias": "{stream=\"stderr\"} stderr", + "color": "#C4162A", + "legend": false, + "linewidth": 2 + }, + { + "$$hashKey": "object:220", + "alias": "{stream=\"stdout\"} stdout", + "color": "#56A64B", + "legend": false, + "linewidth": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(count_over_time(({namespace=\"$namespace\", pod=~\"$pod\"})[$__interval])) by (stream)", + "hide": false, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Count of stderr / stdout historical", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:188", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:189", + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "1m", + "schemaVersion": 25, + "style": "dark", + "tags": [ + "Loki", + "logging" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "loki", + "value": "loki" + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values({namespace=~\".+\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values({namespace=~\".+\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values({namespace=\"$namespace\"}, pod)", + "hide": 0, + "includeAll": true, + "label": "Pod", + "multi": false, + "name": "pod", + "options": [], + "query": "label_values({namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values({namespace=\"$namespace\"}, stream)", + "hide": 0, + "includeAll": true, + "label": "Stream", + "multi": false, + "name": "stream", + "options": [], + "query": "label_values({namespace=\"$namespace\"}, stream)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Search (case insensitive)", + "name": "searchable_pattern", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Application logs", + "uid": "application-logs", + "version": 2, + "description": "Container/Pod Log Dashboard(Loki as DataSource)" + } diff --git a/add-ons/kube-prometheus-stack/dashboard-http-requests.yaml b/add-ons/kube-prometheus-stack/dashboard-http-requests.yaml new file mode 100644 index 0000000..621d4fd --- /dev/null +++ b/add-ons/kube-prometheus-stack/dashboard-http-requests.yaml @@ -0,0 +1,213 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: grafana-dashboard-http-requests + namespace: monitoring +data: + http-requests.json: | + { + "uid": "http-requests", + "title": "HTTP requests", + "editable": true, + "schemaVersion": 41, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "tags": [], + "templating": { + "list": [] + }, + "panels": [ + { + "type": "timeseries", + "title": "Requests by host", + "id": 1, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (RequestHost) (count_over_time({namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" [$__auto]))", + "legendFormat": "{{RequestHost}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 80, + "showPoints": "never", + "stacking": { + "mode": "normal" + }, + "drawStyle": "bars" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Responses by status", + "id": 2, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (DownstreamStatus) (count_over_time({namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" [$__auto]))", + "legendFormat": "{{DownstreamStatus}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 80, + "showPoints": "never", + "stacking": { + "mode": "normal" + }, + "drawStyle": "bars" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "p95 response time by host", + "id": 3, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.95, {namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" | unwrap Duration [$__auto]) by (RequestHost)", + "legendFormat": "{{RequestHost}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ns", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "logs", + "title": "Access log", + "id": 4, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "{namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" | line_format \"{{.DownstreamStatus}} {{.RequestMethod}} {{.RequestHost}}{{.RequestPath}} {{div .Duration 1000000}}ms {{.ClientHost}}\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": false, + "sortOrder": "Descending", + "enableLogDetails": true, + "dedupStrategy": "none", + "showLabels": false + } + } + ], + "annotations": { + "list": [] + } + } diff --git a/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml b/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml new file mode 100644 index 0000000..d1c4dee --- /dev/null +++ b/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml @@ -0,0 +1,985 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: grafana-dashboard-raspberry-pi + namespace: monitoring +data: + raspberry-pi.json: | + { + "uid": "raspberry-pi", + "title": "Raspberry Pi", + "editable": true, + "schemaVersion": 41, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "tags": [], + "templating": { + "list": [ + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(container_cpu_usage_seconds_total, namespace)", + "refId": "ns" + }, + "definition": "label_values(container_cpu_usage_seconds_total, namespace)", + "multi": true, + "includeAll": true, + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Node", + "id": 1, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "stat", + "title": "CPU", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Memory", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Disk /", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "CPU temperature", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_thermal_zone_temp{type=\"cpu-thermal\"}", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "NVMe temperature", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(node_hwmon_temp_celsius{chip=\"nvme_nvme0\"})", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 60 + }, + { + "color": "red", + "value": 70 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Uptime", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "time() - node_boot_time_seconds", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "CPU", + "id": 8, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "legendFormat": "used" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * avg(rate(node_cpu_seconds_total{mode=\"iowait\"}[$__rate_interval]))", + "legendFormat": "iowait" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Memory", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "legendFormat": "used" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemAvailable_bytes", + "legendFormat": "available" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Disk I/O", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(node_disk_read_bytes_total{device=~\"nvme.+n1|mmcblk0\"}[$__rate_interval]))", + "legendFormat": "read" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(node_disk_written_bytes_total{device=~\"nvme.+n1|mmcblk0\"}[$__rate_interval]))", + "legendFormat": "written" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Network", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(node_network_receive_bytes_total{device=~\"eth0|wlan0\"}[$__rate_interval])", + "legendFormat": "{{device}} in" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(node_network_transmit_bytes_total{device=~\"eth0|wlan0\"}[$__rate_interval])", + "legendFormat": "{{device}} out" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Temperatures", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_thermal_zone_temp{type=\"cpu-thermal\"}", + "legendFormat": "cpu" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(node_hwmon_temp_celsius{chip=\"nvme_nvme0\"})", + "legendFormat": "nvme" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Pods", + "id": 13, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "CPU by namespace", + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{namespace}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Memory by namespace", + "id": 15, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=\"\",namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Top pods by CPU", + "id": 16, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "{{namespace}}/{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Top pods by memory", + "id": 17, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (namespace, pod) (container_memory_working_set_bytes{container!=\"\",namespace=~\"$namespace\"}))", + "legendFormat": "{{namespace}}/{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Top pods by disk writes", + "id": 18, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (namespace, pod) (rate(container_fs_writes_bytes_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "{{namespace}}/{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Network by namespace", + "id": 19, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{namespace}} in" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "-sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{namespace}} out" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + } + ], + "annotations": { + "list": [] + } + } diff --git a/add-ons/kube-prometheus-stack/kustomization.yaml b/add-ons/kube-prometheus-stack/kustomization.yaml index 4a09a42..85f302e 100644 --- a/add-ons/kube-prometheus-stack/kustomization.yaml +++ b/add-ons/kube-prometheus-stack/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - dashboard-application-logs.yaml + - dashboard-http-requests.yaml + - dashboard-raspberry-pi.yaml - grafana-admin-sealedsecret.yaml - grafana-db-sealedsecret.yaml - grafana-oidc-sealedsecret.yaml - httproute.yaml - - scrapeconfigs.yaml diff --git a/add-ons/kube-prometheus-stack/scrapeconfigs.yaml b/add-ons/kube-prometheus-stack/scrapeconfigs.yaml deleted file mode 100644 index bcb8470..0000000 --- a/add-ons/kube-prometheus-stack/scrapeconfigs.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# k3s control plane (bound in /etc/rancher/k3s/config.yaml); every endpoint also serves apiserver series. -apiVersion: monitoring.coreos.com/v1alpha1 -kind: ScrapeConfig -metadata: - name: kube-controller-manager - namespace: monitoring -spec: - jobName: kube-controller-manager - scheme: HTTPS - tlsConfig: - insecureSkipVerify: true - authorization: - credentials: - name: kube-prometheus-stack-prometheus-token - key: token - staticConfigs: - - targets: - - 192.168.2.5:10257 - metricRelabelings: - - action: drop - sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* ---- -apiVersion: monitoring.coreos.com/v1alpha1 -kind: ScrapeConfig -metadata: - name: kube-scheduler - namespace: monitoring -spec: - jobName: kube-scheduler - scheme: HTTPS - tlsConfig: - insecureSkipVerify: true - authorization: - credentials: - name: kube-prometheus-stack-prometheus-token - key: token - staticConfigs: - - targets: - - 192.168.2.5:10259 - metricRelabelings: - - action: drop - sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* ---- -apiVersion: monitoring.coreos.com/v1alpha1 -kind: ScrapeConfig -metadata: - name: kube-proxy - namespace: monitoring -spec: - jobName: kube-proxy - staticConfigs: - - targets: - - 192.168.2.5:10249 - metricRelabelings: - - action: drop - sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* diff --git a/add-ons/postgres/cluster.yaml b/add-ons/postgres/cluster.yaml index bfb2b4c..d6f9734 100644 --- a/add-ons/postgres/cluster.yaml +++ b/add-ons/postgres/cluster.yaml @@ -37,9 +37,7 @@ spec: passwordSecret: name: redoost-db postgresql: - # Each app role may only connect to the database of the same name. - # pgweb is read-only and may browse every database except authentik's, - # which holds credentials and session tokens. + # Each app role may only connect to its own database; pgweb to all but authentik. pg_hba: - host sameuser all all scram-sha-256 - host authentik pgweb all reject diff --git a/apps/pgweb/deployment.yaml b/apps/pgweb/deployment.yaml index c4a197a..96dd5da 100644 --- a/apps/pgweb/deployment.yaml +++ b/apps/pgweb/deployment.yaml @@ -16,8 +16,6 @@ spec: app.kubernetes.io/name: pgweb spec: containers: - # Only bookmarked databases can be connected to, picked on the - # connect screen; there is one bookmark per app database. - args: - --bookmarks-dir=/bookmarks - --bookmarks-only diff --git a/apps/pgweb/networkpolicy.yaml b/apps/pgweb/networkpolicy.yaml index a1e47cd..63b27f3 100644 --- a/apps/pgweb/networkpolicy.yaml +++ b/apps/pgweb/networkpolicy.yaml @@ -4,8 +4,7 @@ metadata: name: pgweb namespace: pgweb spec: - # Only traefik may reach pgweb, so authentik can't be bypassed from inside - # the cluster, and pgweb may only reach postgres. + # Only traefik may reach pgweb, so authentik can't be bypassed. egress: - ports: - port: 53 diff --git a/apps/wireguard/deployment.yaml b/apps/wireguard/deployment.yaml index 18de01e..e71a317 100644 --- a/apps/wireguard/deployment.yaml +++ b/apps/wireguard/deployment.yaml @@ -18,13 +18,11 @@ spec: containers: - args: - | - # The image points iptables at the legacy backend, which the - # node's kernel doesn't have. Use nftables, as the node does. + # The node's kernel only has nftables, the image defaults to legacy iptables. for b in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do ln -sf xtables-nft-multi "/usr/sbin/$b" done - # Only used to skip the first-run setup; password login is - # disabled in favour of authentik. + # Only skips the setup wizard; password login is disabled. INIT_PASSWORD="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')" export INIT_PASSWORD exec /usr/bin/dumb-init node server/index.mjs @@ -83,11 +81,7 @@ spec: initContainers: - args: - | - # Rules in the pod's own network namespace, never the node's. - # VPN clients may reach the home LAN, but not this pod (the web - # UI), the pod network or the service network. The mangle table - # runs before wg-easy's own filter rules, so its per-client - # firewall can't open these up. + # VPN clients may reach the LAN, not this pod or the cluster networks. set -eu iptables-nft -t mangle -A PREROUTING -i wg0 -m addrtype --dst-type LOCAL -j DROP iptables-nft -t mangle -A PREROUTING -i wg0 -d 10.42.0.0/16 -j DROP diff --git a/argocd-apps/add-ons/kube-prometheus-stack.yaml b/argocd-apps/add-ons/kube-prometheus-stack.yaml index d50f188..52f54da 100644 --- a/argocd-apps/add-ons/kube-prometheus-stack.yaml +++ b/argocd-apps/add-ons/kube-prometheus-stack.yaml @@ -17,35 +17,38 @@ spec: values: | alertmanager: enabled: false + # Only node-exporter and cAdvisor are scraped. + defaultRules: + create: false kubelet: serviceMonitor: - # The first rule is the chart's default, as lists are replaced; k3s also serves apiserver series here. - metricRelabelings: - - action: drop - sourceLabels: [__name__, le] - regex: (csi_operations|storage_operation_duration)_seconds_bucket;(0.25|2.5|15|25|120|600)(\.0)? - - action: drop + interval: 60s + kubelet: false + probes: false + cAdvisorMetricRelabelings: + - action: keep sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* - # Scraped through ScrapeConfigs since ArgoCD ignores Endpoints; enabled for dashboards and rules. + regex: container_(cpu_usage_seconds_total|memory_working_set_bytes|fs_(reads|writes)_bytes_total|network_(receive|transmit)_bytes_total) + - action: drop + sourceLabels: [__name__, interface] + regex: container_network_.+;(br-|cni|docker|flannel|lo|tailscale|tun|veth|wg|wlan).* + - action: drop + sourceLabels: [id, pod] + regex: .+; + kubeApiServer: + enabled: false kubeControllerManager: - service: - enabled: false - serviceMonitor: - enabled: false + enabled: false kubeScheduler: - service: - enabled: false - serviceMonitor: - enabled: false + enabled: false kubeProxy: - service: - enabled: false - serviceMonitor: - enabled: false - # k3s runs on SQLite (kine), there is no etcd to scrape. + enabled: false kubeEtcd: enabled: false + coreDns: + enabled: false + kubeStateMetrics: + enabled: false prometheusOperator: admissionWebhooks: certManager: @@ -65,15 +68,15 @@ spec: requests: cpu: 10m memory: 32Mi - kube-state-metrics: - resources: - limits: - cpu: 200m - memory: 256Mi - requests: - cpu: 20m - memory: 64Mi + serviceMonitor: + selfMonitor: false prometheus-node-exporter: + # The first two are the chart's defaults, as lists are replaced. + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$ + - --collector.netclass.ignored-devices=^(veth.*|br-.*|docker0|cni0|lo)$ + - --collector.netdev.device-exclude=^(veth.*|br-.*|docker0|cni0|lo)$ resources: limits: cpu: 200m @@ -82,7 +85,10 @@ spec: cpu: 20m memory: 32Mi prometheus: + serviceMonitor: + selfMonitor: false prometheusSpec: + scrapeInterval: 60s retention: 90d retentionSize: 45GB # Pick up monitors and rules from every namespace, not just this release. @@ -114,6 +120,8 @@ spec: type: loki uid: loki url: http://loki.monitoring.svc.cluster.local:3100 + isDefault: true + defaultDashboardsEnabled: false envValueFrom: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: secretKeyRef: @@ -155,7 +163,13 @@ spec: requests: cpu: 50m memory: 256Mi + serviceMonitor: + enabled: false sidecar: + datasources: + alertmanager: + enabled: false + isDefaultDatasource: false dashboards: searchNamespace: ALL resources: diff --git a/argocd-apps/add-ons/traefik.yaml b/argocd-apps/add-ons/traefik.yaml index 32f722d..4ba9408 100644 --- a/argocd-apps/add-ons/traefik.yaml +++ b/argocd-apps/add-ons/traefik.yaml @@ -15,6 +15,13 @@ spec: - chart: traefik helm: values: | + # Query strings are dropped, as OAuth callbacks carry codes. + accessLog: + enabled: true + format: json + fields: + queryParameters: + defaultMode: drop ingressClass: enabled: false providers: