From e913f42443707a5e53b627304d3990b8f4b2dba2 Mon Sep 17 00:00:00 2001 From: Karan Parashar Date: Sat, 10 Oct 2026 12:45:21 +0530 Subject: [PATCH] feat(monitoring): Scrape node and pod metrics, add logs and Pi dashboards --- add-ons/authentik/blueprint-argocd.yaml | 1 - add-ons/authentik/blueprint-outpost.yaml | 1 - .../authentik/blueprint-wireguard-admin.yaml | 5 +- .../kube-prometheus-stack/dashboard-logs.yaml | 422 ++++++++ .../dashboard-raspberry-pi.yaml | 985 ++++++++++++++++++ .../kube-prometheus-stack/kustomization.yaml | 3 +- .../kube-prometheus-stack/scrapeconfigs.yaml | 59 -- add-ons/postgres/cluster.yaml | 4 +- apps/pgweb/deployment.yaml | 2 - apps/pgweb/networkpolicy.yaml | 3 +- apps/wireguard/deployment.yaml | 12 +- .../add-ons/kube-prometheus-stack.yaml | 72 +- argocd-apps/add-ons/traefik.yaml | 7 + 13 files changed, 1466 insertions(+), 110 deletions(-) create mode 100644 add-ons/kube-prometheus-stack/dashboard-logs.yaml create mode 100644 add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml delete mode 100644 add-ons/kube-prometheus-stack/scrapeconfigs.yaml diff --git a/add-ons/authentik/blueprint-argocd.yaml b/add-ons/authentik/blueprint-argocd.yaml index ffe7ed1..041fbf3 100644 --- a/add-ons/authentik/blueprint-argocd.yaml +++ b/add-ons/authentik/blueprint-argocd.yaml @@ -9,7 +9,6 @@ data: metadata: name: ArgoCD entries: - # Public client: the CLI can't keep a secret, so ArgoCD uses PKCE instead. - model: authentik_providers_oauth2.oauth2provider id: argocd-provider identifiers: diff --git a/add-ons/authentik/blueprint-outpost.yaml b/add-ons/authentik/blueprint-outpost.yaml index 0ddc35b..5cb6913 100644 --- a/add-ons/authentik/blueprint-outpost.yaml +++ b/add-ons/authentik/blueprint-outpost.yaml @@ -24,7 +24,6 @@ data: identifiers: path: mounted/cm-blueprint-qbittorrent/qbittorrent.yaml required: true - # Serves forward auth for the apps behind the authentik middleware. - model: authentik_outposts.outpost identifiers: managed: goauthentik.io/outposts/embedded diff --git a/add-ons/authentik/blueprint-wireguard-admin.yaml b/add-ons/authentik/blueprint-wireguard-admin.yaml index acbab2b..0ca26b6 100644 --- a/add-ons/authentik/blueprint-wireguard-admin.yaml +++ b/add-ons/authentik/blueprint-wireguard-admin.yaml @@ -9,8 +9,7 @@ data: metadata: name: WireGuard admin entries: - # wg-easy rejects logins unless email_verified is true; the default email - # mapping always sends false. + # wg-easy requires email_verified, which the default mapping sends as false. - model: authentik_providers_oauth2.scopemapping id: verified-email identifiers: @@ -52,7 +51,7 @@ data: name: WireGuard Admin provider: !KeyOf wireguard-admin-provider meta_launch_url: https://wg-admin.whyredfire.dev - # wg-easy makes every user it registers an admin, so only admins may log in. + # wg-easy makes every user an admin. - model: authentik_policies.policybinding identifiers: order: 0 diff --git a/add-ons/kube-prometheus-stack/dashboard-logs.yaml b/add-ons/kube-prometheus-stack/dashboard-logs.yaml new file mode 100644 index 0000000..d1b7c08 --- /dev/null +++ b/add-ons/kube-prometheus-stack/dashboard-logs.yaml @@ -0,0 +1,422 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: grafana-dashboard-logs + namespace: monitoring +data: + logs.json: | + { + "uid": "logs", + "title": "Logs", + "editable": true, + "schemaVersion": 41, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "tags": [], + "templating": { + "list": [ + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "type": 1, + "label": "namespace", + "refId": "ns" + }, + "definition": "label_values(namespace)", + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + }, + { + "name": "app", + "label": "App", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "type": 1, + "label": "app", + "stream": "{namespace=~\"$namespace\"}", + "refId": "app" + }, + "definition": "label_values({namespace=~\"$namespace\"}, app)", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + } + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Log volume", + "id": 1, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (app) (count_over_time({namespace=~\"$namespace\", app=~\"$app\"} |= \"$search\" [$__auto]))", + "legendFormat": "{{app}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 80, + "showPoints": "never", + "stacking": { + "mode": "normal" + }, + "drawStyle": "bars" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Error lines", + "id": 2, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (app) (count_over_time({namespace=~\"$namespace\", app=~\"$app\"} |= \"$search\" |~ \"(?i)(error|exception|panic|fatal)\" [$__auto]))", + "legendFormat": "{{app}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 80, + "showPoints": "never", + "stacking": { + "mode": "normal" + }, + "drawStyle": "bars" + } + }, + "overrides": [] + } + }, + { + "type": "logs", + "title": "Logs", + "id": 3, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=~\"$namespace\", app=~\"$app\"} |= \"$search\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true, + "dedupStrategy": "none", + "showLabels": false + } + }, + { + "type": "row", + "title": "HTTP requests (Traefik)", + "id": 4, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Requests by host", + "id": 5, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (RequestHost) (count_over_time({namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" [$__auto]))", + "legendFormat": "{{RequestHost}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 80, + "showPoints": "never", + "stacking": { + "mode": "normal" + }, + "drawStyle": "bars" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Responses by status", + "id": 6, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (DownstreamStatus) (count_over_time({namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" [$__auto]))", + "legendFormat": "{{DownstreamStatus}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 80, + "showPoints": "never", + "stacking": { + "mode": "normal" + }, + "drawStyle": "bars" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "p95 response time by host", + "id": 7, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.95, {namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" | unwrap Duration [$__auto]) by (RequestHost)", + "legendFormat": "{{RequestHost}}", + "queryType": "range" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ns", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "logs", + "title": "Access log", + "id": 8, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 42 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "expr": "{namespace=\"traefik\"} | json | __error__=\"\" | RequestHost != \"\" | line_format \"{{.DownstreamStatus}} {{.RequestMethod}} {{.RequestHost}}{{.RequestPath}} {{div .Duration 1000000}}ms {{.ClientHost}}\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": false, + "sortOrder": "Descending", + "enableLogDetails": true, + "dedupStrategy": "none", + "showLabels": false + } + } + ], + "annotations": { + "list": [] + } + } diff --git a/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml b/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml new file mode 100644 index 0000000..d1c4dee --- /dev/null +++ b/add-ons/kube-prometheus-stack/dashboard-raspberry-pi.yaml @@ -0,0 +1,985 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: grafana-dashboard-raspberry-pi + namespace: monitoring +data: + raspberry-pi.json: | + { + "uid": "raspberry-pi", + "title": "Raspberry Pi", + "editable": true, + "schemaVersion": 41, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "tags": [], + "templating": { + "list": [ + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(container_cpu_usage_seconds_total, namespace)", + "refId": "ns" + }, + "definition": "label_values(container_cpu_usage_seconds_total, namespace)", + "multi": true, + "includeAll": true, + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Node", + "id": 1, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "stat", + "title": "CPU", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Memory", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Disk /", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "CPU temperature", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_thermal_zone_temp{type=\"cpu-thermal\"}", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "NVMe temperature", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(node_hwmon_temp_celsius{chip=\"nvme_nvme0\"})", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 60 + }, + { + "color": "red", + "value": 70 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Uptime", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "time() - node_boot_time_seconds", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0 + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "CPU", + "id": 8, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "legendFormat": "used" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 * avg(rate(node_cpu_seconds_total{mode=\"iowait\"}[$__rate_interval]))", + "legendFormat": "iowait" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Memory", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "legendFormat": "used" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemAvailable_bytes", + "legendFormat": "available" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Disk I/O", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(node_disk_read_bytes_total{device=~\"nvme.+n1|mmcblk0\"}[$__rate_interval]))", + "legendFormat": "read" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(node_disk_written_bytes_total{device=~\"nvme.+n1|mmcblk0\"}[$__rate_interval]))", + "legendFormat": "written" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Network", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(node_network_receive_bytes_total{device=~\"eth0|wlan0\"}[$__rate_interval])", + "legendFormat": "{{device}} in" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(node_network_transmit_bytes_total{device=~\"eth0|wlan0\"}[$__rate_interval])", + "legendFormat": "{{device}} out" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Temperatures", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_thermal_zone_temp{type=\"cpu-thermal\"}", + "legendFormat": "cpu" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(node_hwmon_temp_celsius{chip=\"nvme_nvme0\"})", + "legendFormat": "nvme" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Pods", + "id": 13, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "CPU by namespace", + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{namespace}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Memory by namespace", + "id": 15, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (namespace) (container_memory_working_set_bytes{container!=\"\",namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Top pods by CPU", + "id": 16, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "{{namespace}}/{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Top pods by memory", + "id": 17, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (namespace, pod) (container_memory_working_set_bytes{container!=\"\",namespace=~\"$namespace\"}))", + "legendFormat": "{{namespace}}/{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Top pods by disk writes", + "id": 18, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (namespace, pod) (rate(container_fs_writes_bytes_total{container!=\"\",namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "{{namespace}}/{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Network by namespace", + "id": 19, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{namespace}} in" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "-sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{namespace}} out" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + } + } + ], + "annotations": { + "list": [] + } + } diff --git a/add-ons/kube-prometheus-stack/kustomization.yaml b/add-ons/kube-prometheus-stack/kustomization.yaml index 4a09a42..c06c211 100644 --- a/add-ons/kube-prometheus-stack/kustomization.yaml +++ b/add-ons/kube-prometheus-stack/kustomization.yaml @@ -1,8 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - dashboard-logs.yaml + - dashboard-raspberry-pi.yaml - grafana-admin-sealedsecret.yaml - grafana-db-sealedsecret.yaml - grafana-oidc-sealedsecret.yaml - httproute.yaml - - scrapeconfigs.yaml diff --git a/add-ons/kube-prometheus-stack/scrapeconfigs.yaml b/add-ons/kube-prometheus-stack/scrapeconfigs.yaml deleted file mode 100644 index bcb8470..0000000 --- a/add-ons/kube-prometheus-stack/scrapeconfigs.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# k3s control plane (bound in /etc/rancher/k3s/config.yaml); every endpoint also serves apiserver series. -apiVersion: monitoring.coreos.com/v1alpha1 -kind: ScrapeConfig -metadata: - name: kube-controller-manager - namespace: monitoring -spec: - jobName: kube-controller-manager - scheme: HTTPS - tlsConfig: - insecureSkipVerify: true - authorization: - credentials: - name: kube-prometheus-stack-prometheus-token - key: token - staticConfigs: - - targets: - - 192.168.2.5:10257 - metricRelabelings: - - action: drop - sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* ---- -apiVersion: monitoring.coreos.com/v1alpha1 -kind: ScrapeConfig -metadata: - name: kube-scheduler - namespace: monitoring -spec: - jobName: kube-scheduler - scheme: HTTPS - tlsConfig: - insecureSkipVerify: true - authorization: - credentials: - name: kube-prometheus-stack-prometheus-token - key: token - staticConfigs: - - targets: - - 192.168.2.5:10259 - metricRelabelings: - - action: drop - sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* ---- -apiVersion: monitoring.coreos.com/v1alpha1 -kind: ScrapeConfig -metadata: - name: kube-proxy - namespace: monitoring -spec: - jobName: kube-proxy - staticConfigs: - - targets: - - 192.168.2.5:10249 - metricRelabelings: - - action: drop - sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* diff --git a/add-ons/postgres/cluster.yaml b/add-ons/postgres/cluster.yaml index bfb2b4c..d6f9734 100644 --- a/add-ons/postgres/cluster.yaml +++ b/add-ons/postgres/cluster.yaml @@ -37,9 +37,7 @@ spec: passwordSecret: name: redoost-db postgresql: - # Each app role may only connect to the database of the same name. - # pgweb is read-only and may browse every database except authentik's, - # which holds credentials and session tokens. + # Each app role may only connect to its own database; pgweb to all but authentik. pg_hba: - host sameuser all all scram-sha-256 - host authentik pgweb all reject diff --git a/apps/pgweb/deployment.yaml b/apps/pgweb/deployment.yaml index c4a197a..96dd5da 100644 --- a/apps/pgweb/deployment.yaml +++ b/apps/pgweb/deployment.yaml @@ -16,8 +16,6 @@ spec: app.kubernetes.io/name: pgweb spec: containers: - # Only bookmarked databases can be connected to, picked on the - # connect screen; there is one bookmark per app database. - args: - --bookmarks-dir=/bookmarks - --bookmarks-only diff --git a/apps/pgweb/networkpolicy.yaml b/apps/pgweb/networkpolicy.yaml index a1e47cd..63b27f3 100644 --- a/apps/pgweb/networkpolicy.yaml +++ b/apps/pgweb/networkpolicy.yaml @@ -4,8 +4,7 @@ metadata: name: pgweb namespace: pgweb spec: - # Only traefik may reach pgweb, so authentik can't be bypassed from inside - # the cluster, and pgweb may only reach postgres. + # Only traefik may reach pgweb, so authentik can't be bypassed. egress: - ports: - port: 53 diff --git a/apps/wireguard/deployment.yaml b/apps/wireguard/deployment.yaml index 18de01e..e71a317 100644 --- a/apps/wireguard/deployment.yaml +++ b/apps/wireguard/deployment.yaml @@ -18,13 +18,11 @@ spec: containers: - args: - | - # The image points iptables at the legacy backend, which the - # node's kernel doesn't have. Use nftables, as the node does. + # The node's kernel only has nftables, the image defaults to legacy iptables. for b in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do ln -sf xtables-nft-multi "/usr/sbin/$b" done - # Only used to skip the first-run setup; password login is - # disabled in favour of authentik. + # Only skips the setup wizard; password login is disabled. INIT_PASSWORD="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')" export INIT_PASSWORD exec /usr/bin/dumb-init node server/index.mjs @@ -83,11 +81,7 @@ spec: initContainers: - args: - | - # Rules in the pod's own network namespace, never the node's. - # VPN clients may reach the home LAN, but not this pod (the web - # UI), the pod network or the service network. The mangle table - # runs before wg-easy's own filter rules, so its per-client - # firewall can't open these up. + # VPN clients may reach the LAN, not this pod or the cluster networks. set -eu iptables-nft -t mangle -A PREROUTING -i wg0 -m addrtype --dst-type LOCAL -j DROP iptables-nft -t mangle -A PREROUTING -i wg0 -d 10.42.0.0/16 -j DROP diff --git a/argocd-apps/add-ons/kube-prometheus-stack.yaml b/argocd-apps/add-ons/kube-prometheus-stack.yaml index d50f188..52f54da 100644 --- a/argocd-apps/add-ons/kube-prometheus-stack.yaml +++ b/argocd-apps/add-ons/kube-prometheus-stack.yaml @@ -17,35 +17,38 @@ spec: values: | alertmanager: enabled: false + # Only node-exporter and cAdvisor are scraped. + defaultRules: + create: false kubelet: serviceMonitor: - # The first rule is the chart's default, as lists are replaced; k3s also serves apiserver series here. - metricRelabelings: - - action: drop - sourceLabels: [__name__, le] - regex: (csi_operations|storage_operation_duration)_seconds_bucket;(0.25|2.5|15|25|120|600)(\.0)? - - action: drop + interval: 60s + kubelet: false + probes: false + cAdvisorMetricRelabelings: + - action: keep sourceLabels: [__name__] - regex: (apiserver|etcd|kine)_.* - # Scraped through ScrapeConfigs since ArgoCD ignores Endpoints; enabled for dashboards and rules. + regex: container_(cpu_usage_seconds_total|memory_working_set_bytes|fs_(reads|writes)_bytes_total|network_(receive|transmit)_bytes_total) + - action: drop + sourceLabels: [__name__, interface] + regex: container_network_.+;(br-|cni|docker|flannel|lo|tailscale|tun|veth|wg|wlan).* + - action: drop + sourceLabels: [id, pod] + regex: .+; + kubeApiServer: + enabled: false kubeControllerManager: - service: - enabled: false - serviceMonitor: - enabled: false + enabled: false kubeScheduler: - service: - enabled: false - serviceMonitor: - enabled: false + enabled: false kubeProxy: - service: - enabled: false - serviceMonitor: - enabled: false - # k3s runs on SQLite (kine), there is no etcd to scrape. + enabled: false kubeEtcd: enabled: false + coreDns: + enabled: false + kubeStateMetrics: + enabled: false prometheusOperator: admissionWebhooks: certManager: @@ -65,15 +68,15 @@ spec: requests: cpu: 10m memory: 32Mi - kube-state-metrics: - resources: - limits: - cpu: 200m - memory: 256Mi - requests: - cpu: 20m - memory: 64Mi + serviceMonitor: + selfMonitor: false prometheus-node-exporter: + # The first two are the chart's defaults, as lists are replaced. + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$ + - --collector.netclass.ignored-devices=^(veth.*|br-.*|docker0|cni0|lo)$ + - --collector.netdev.device-exclude=^(veth.*|br-.*|docker0|cni0|lo)$ resources: limits: cpu: 200m @@ -82,7 +85,10 @@ spec: cpu: 20m memory: 32Mi prometheus: + serviceMonitor: + selfMonitor: false prometheusSpec: + scrapeInterval: 60s retention: 90d retentionSize: 45GB # Pick up monitors and rules from every namespace, not just this release. @@ -114,6 +120,8 @@ spec: type: loki uid: loki url: http://loki.monitoring.svc.cluster.local:3100 + isDefault: true + defaultDashboardsEnabled: false envValueFrom: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: secretKeyRef: @@ -155,7 +163,13 @@ spec: requests: cpu: 50m memory: 256Mi + serviceMonitor: + enabled: false sidecar: + datasources: + alertmanager: + enabled: false + isDefaultDatasource: false dashboards: searchNamespace: ALL resources: diff --git a/argocd-apps/add-ons/traefik.yaml b/argocd-apps/add-ons/traefik.yaml index 32f722d..4ba9408 100644 --- a/argocd-apps/add-ons/traefik.yaml +++ b/argocd-apps/add-ons/traefik.yaml @@ -15,6 +15,13 @@ spec: - chart: traefik helm: values: | + # Query strings are dropped, as OAuth callbacks carry codes. + accessLog: + enabled: true + format: json + fields: + queryParameters: + defaultMode: drop ingressClass: enabled: false providers: