feat(add-ons): Add observability stack with kube-prometheus-stack, loki and alloy

add-ons/authentik/blueprints.yaml

@@ -67,3 +67,33 @@ data:
           domain: authentik-default
         attrs:
           flow_authentication: !KeyOf google-flow
+  grafana.yaml: |
+    version: 1
+    metadata:
+      name: Grafana
+    entries:
+      - model: authentik_providers_oauth2.oauth2provider
+        id: grafana-provider
+        identifiers:
+          name: grafana
+        attrs:
+          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          client_type: confidential
+          client_id: grafana
+          client_secret: !Env GRAFANA_CLIENT_SECRET
+          redirect_uris:
+            - matching_mode: strict
+              url: https://grafana.whyredfire.dev/login/generic_oauth
+          signing_key: !Find [authentik_crypto.certificatekeypair, [name, authentik Self-signed Certificate]]
+          property_mappings:
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]]
+            - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-profile]]
+      - model: authentik_core.application
+        identifiers:
+          slug: grafana
+        attrs:
+          name: Grafana
+          provider: !KeyOf grafana-provider
+          meta_launch_url: https://grafana.whyredfire.dev

add-ons/authentik/sealed-secret.yaml

@@ -11,6 +11,7 @@ spec:
     AUTHENTIK_SECRET_KEY: AgAHHDs8mbJRc45DFgap2VUZn/F+m59hm9u2lebuJv70HSBxLLsHEn8nXLGUO3b2ScyPt1Qi2wbmRDENTjOHOYHtAQAjwV25r6RUiZ5TosRH2KhGx5N6gceorYQyk18l2CkKv4ObXjeGLJA0EqMarmFWiT24SFZdtIrOvWygLxg4XGful2h1Fw5uXK9PiUHgptrp/G5M5Htv0bvtD6m/FrgOlK6C6tK1D4vLL3VLX3/SceqNWVDIJmJao5JKO4IZTQxN8TN5k9K2CGrtOUhjCn4vLbYQKTPhiiidoQaipQ9FR+4KJMO0y8FbIy+3q7C/UZPyOTOu/2zI4t5h4TJY1+477A80GTLpE0aCBDpmFwgZY728wbTW4NyfIm15u0Um7OXhnJkXC1EqDbvlK4HYHiSDUjGChvlfMEY+D1datDlF1fEuIaeXtTqowpz3uy9qIEa0f1emS2J8TjIrKH3oW3mfmrJKSiySnCX1704ZPlwOJfnF5Z2vM6Oqo/mWHa6PSwPPRxSxEKVtSqD4KynzrUmx3GkrUs+7tntepA3yQoiDjyUyE71FwhJXO4sE3XcRGDUf4rB8CFO4+NgTbBAqFzNKYG32GIDhGjbMxBAJQMNXZhVcKceiaXf7G8LuaOEw39BkdtiFCdK3uXl9xaQPuStRsOYYQR2Dt0vt3AvDGjAW+McauCbYUD3rcAtVMo+rxEbejrotojWnb1OpGX0JqD5gHC68qlqY6BYkDbWpYojoKWJHuHlYxHIm8k7ghSq6npw94R526bxripNtcNq4Lh1NeM55LKruOEGr7n0dFcyAmQ==
     GOOGLE_CLIENT_ID: AgCV2Rwe/bHbkQ3nkkIs4EFTbsEqQWRaWlXa/fHzkDyNKpd0CTOgqVb6Rd2KYyvoT9iDR0xt/8reLPd9e7B3G3V1oVFV7G1XiErBmzvxr6591+tXzo8JLVpIy0363R87idi2xIKGVii+G5aJvxxhua8IRsNfi4SRtoqpDzjgNbFtm5UDhhlCzcRKkm0zHgLw83d4Uwszu9X9II2A4FuMPD6KvsTFrgHevR1v3OerKl5kvTlUx0O3XEg5mnNUVk46J+D9wfg58bIlgUhMV8NtfKa8I5zmRKhCJySks1zlCgIudCLX7wPYupFgt0nrZjJGx7a546ntPdMVE4IJ+VhlQ94gjdWsVFm07espaQLph9T4CZDKxb6TwyNswhfFxH1Z9auyFf3Cfk2wx5GChNOL0mZ8Zrzcw008c8IUShF8je8JZ60teBffbBNao7XN6JM6a4ac5aKnNwFch9IJUU1qkvcJaZk5hAkeGO9uHrICb6lkr5Rj7t4h9l4mvgfIKXP5YCCKZ+yqrSI8Jpkch/cB0Sphgp7m7QXgGwmWRgdKEk1rP7ioIfoF66mLsfgvOkc+1ySvPgW/rCLbHwzMqJn1Fkk/afbRoWp/guFj1cOY9MrEQIn8ftugeV/0xWiL+y5RyJm7QZc1z5fSj/NTrVCZV0LhBHttPZzoGGWEqjLYQnyJ8p38lK7EU6Pmk4WrEMj7mmdYHEijyPhab+b7guOc+ULbPT7k7sBHXbKrTTUJa09jIqSS4jl2PNrzPCPDIvRpG/cMgNklhNPfvpDQcLdVRA7bsf7navZvDsQ=
     GOOGLE_CLIENT_SECRET: AgDNFpqV5LGekTMvOBtPE9c6ZVPtT6wN+nCFRhZnvBKIz4I7U9jzisEN7yhc4ud22XJXpS5oR8hN4H+V6Js/bCZMDJefYcdstmHku14kUvoLRUtwPV9tdjiihmb57VLTnT1s1OzbnOqZPwXoCAmGAPLDgZrghBi0id21GgjU41jCHDJCcH1j9RHhE20JwsVG2uz9Tcp6MVYh94aq53V4//bvFXz4xQgVlN8c1jJ/W4rrNM9vmYdMwKyB5509oVWJigowsRlji0FXurR8ovz1AXpMuGj6hUGxLp0uHsqz7FV/0lsd4B2BR9cVRChAR8zDCiiKb4TH2Kmn676/Z+gtJ8vcOQg/WDT1ENzOr6dyndCLVxJvJuvl1EqinzhSqJjUy11HxqULApKKbX1iRJIAIGb6dpeKVoHDKg5xtcWGwaD6epG/tz20agkWBMCFNq5aS80npz28W5GeOQ8aF4X/rpjuEHVLrN+n0pgyS4wjwniB6UJMYRIYk45r0ICUtqyFjOiNzpU70NsPmEjDyItW1EVo+6tb0lRmOI5apm/9yayZXjFISwTAFPdBMrQL+ij4AGw4HMWMmN0aqDO+Kd1c9Jg6wUKluN2Lb6CaxehWeSJiz2wTEhEz8KBkInfezgSekNkw4wjJiBEZ4/M3G6Y/C61ekuVFarRc87cY+S3SomVZDGQcPysfQlieMVpHlfXq/CSDgHhZV3Q+S/wqpy5ZcfsyaU+v4PmvyEc12MZxQ1RGngVJow==
+    GRAFANA_CLIENT_SECRET: AgBSE0QjGwtnyAjO/cy/TBKg+fuyKMSRgeQM/i7tPi9QcLpDjEHquGKmdBHUaeSaon3LRH3CPYQJ61T4d4xsp7EXzCM47MhO/rJ+6Xx+dGiIg071Ijj1qOBYszMNPXLZCst/9EfDEBRXQYmtr8X5UfcyXv8RHewvuqxcwy8qzsPsP5G/CpchGWlZRV3x+R+yCjVmTP7VuatPAltvGuAHsAv3Gh1aEXMJ1F3SsW/SP025V/bMiz1wKWe7/bkqJnXjLkNXWI/lpwzYWQl4/9v1Oiiw+1osH6fHNzXM6RA0ei1/FmlRwS9EqnUnl9hGE76GSllhwU9L6K9boJrKRnZIP6xBOJfrqimZciutTfiu6urghyimVACafQLWkeEB8m/zX1CSxVQ5M95n/dtdlBsWofMVyEtoeTip1HXZF1iee4yMv2yEg8yNSWmnb87fSYIMwpMiv5dYQTLg2WslnEe0DxcnyH+O0/MsmWHFz2f/Z4J6h+kWwMEv1pt8l+xkgbUthGGDx+HIcU9SZ/jyZjnzTt4i4BIbejAoXzMmm6q4zE0BD+9AbAQvEDOg6DW1TIu2mupoKVlqYljNE/9zVOAIBcFL0cJ4HNnW13+ufS4KoptZE3Rfv1tpU6g2MZ4y+QBCOczd+5HgXMtv3P9OweZg/l/LRdbFyH3OZviW/4suJFNbWZNgGUlqT/xyk/C5rn7uGSjSinp0lwkAqfxx7To/KP0jE7hJkVOH1vh+7COpFABJEx/cLAXAaIaVJsio38p5i4s+SMEq85Ew3uI46YKD8ljY7AQN3k66U7xQkLgjAT170Q==
   template:
     metadata:
       name: authentik-secrets

add-ons/kube-prometheus-stack/grafana-admin-sealedsecret.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: grafana-admin
+  namespace: monitoring
+spec:
+  encryptedData:
+    admin-password: AgAvoctRn9pH4kx3i0Q75rwFQYuAavZ7/1sgIHzmuOhzR8qPJVLeMwgmbZHz/dcNwD59tKpk1WmzrazUSdTAgkAKCs+CsxXxmxYoaDx+tUEMxHzpUKtW3On6a9YMzhp1e7iTewSMQVW+JZxnubxgT8Ws5N6iDlb/SVKGqrGMMk0436L4qRrXJvtmFRbcjn0h6dZHjuBNDBsGXh5TJCk07IIDYkaCpkZoJvH4Brf3Zla0jgRI3FQ2qFygButR1HLZQ6n7CPq9rhKhzSt3NWktV4zuU3GnZXgk8h9TML09UVEfv8W3NkKmIoXb7DewMhTbtcGw61eV6oCAERv0G2lVIBUQ6Zrfj942QB3KKvXLWzKNlCOMgV/JJUtEFw+lnsB9CLJzEl7eSodsIIFQIaby4iEozjaB2QtWtrROmR7mQDEqk1N0AUHx8oiuJFtJDeTn/0dzswPT+SRWlYoGr54uxHo0nFwkAZyEQehdrsd0Tor9BlXYt510fTSC4VZKHjFbDggyLPn+rmNrShQKB8yfeouovR/SaMa9LG0GpGR51dmDmtH64pN8ukCvMyjXrttD8qiArj+ysywTA2Fge61m1bw4dhen5v7HUrfZW5TxualndhhcUA1VWB9rljZvCqrzeYAwbdj36uTC37mqqhwa36SNKG/AHjKBJ0968TQRgf+DvP+xyIeqFk/tBA4FIfYkDM1jHrSlydWQ7vPca6pbXFtTVA6xUYpqraeCdcIK1n9mxFBG513W
+    admin-user: AgAzRobKF4UVBc21QSAN42jmMSULMTO5F6wkOYIBqgUMyVAdSD1yW+9KP8s7Bu0+1efiqeAJGtB74IJ/8Py7szgHVv1jqHpKz9hqd8A+nJjAhAXvCaERMzM5KZu1ArAmRX6X6H2OdveOgii8la5mrOBDOXVFiG7AMYzjlP+Ia5H/5HBHsEAEKUUNZF1DCznOS448dGftyKK8XEDkCca9B62rFyV+fjEhnTVkvqlg++wgCv/YqHOWzT1uSXLtOBQzoTiYJM6D1oC4n8+xuQx6o7Z/sy6K1Cc5Ypx0ENPpuPwrIUgfpWt84XgKKM94tzN3szFXrlZe2uGkh5gNTEiSnjRa5azsGU/pp8uNV01FUwCORKbWtvyHm+mLM3V1qMhlQpenF92pOVlsMspvxMMaE9iWz5CaQl7QWEH25rjrlI1FGIQx5tQwZrOD2u7qRZ1xv1rsG/GfvKN3tTj7HkhbHL+OIkJHxEx4Doc6yoW57M8sV3b/18TuUWuRazBxn9RkNhcEd/CP/ZhKW13OhMm/U1fuzPDGqgWHoz+jVM7XCb9/p+X0vxUAt2ZiOAc1n8tGDglxkSYhoZX6agn6fviSaEWskalqpisiSpvpzQra/HxoyWqNG4+ovHiRM1yCdSxdsZoziVuVdnjP2wDV525nuU+38uDxMgAxWEigEFr5sOb5LFKKC+nnSwxaQ/+vk4NPJGqTuu9tGA==
+  template:
+    metadata:
+      name: grafana-admin
+      namespace: monitoring

add-ons/kube-prometheus-stack/grafana-db-sealedsecret.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: grafana-db
+  namespace: monitoring
+spec:
+  encryptedData:
+    password: AgCHwt3pHs+dvK/Dg3/PlRvDnC+LjOg42GXKVl61+wXcJdeIc2dtAHLnu1/OOyOOuIYFvovgS02GVH2RmWQ5kkfZ/lD7XRTJxmYBsBiiYqU5rMENy+EJ4bBLcDw7AoPZBowR3y+nl49qHSuCqPWvLwpQ4hh9OXtB0XpEOD8Vqdpr7t3reEMA0T3xd8TrdriNo2sHro5v8o8qMJvZ/6QGmqZWr0yOPMuNSfm4cTaJLCL/MsYQRsUuJ4+1+LLoldrNjSalYqhamQvA6UsVOnuH0Ajp8rj7lnbyf/PbUx+cvqT+4IkDxDnDffdN66Q5s+oND+fMEM64QhFBP7KPrw1rWQMTC+awe5mjf37h0J1JFtnty4p4gJ6eXjc2wQVleuxDzZP9m26bp0Lf00cMTxxWDSHeZsKg4aQ0aqsSKYyXg2i2Nu/UEX/NBrNGTCFNk8+QIGowYr8KJkgC/WmaUNBLxpiNW6YIPwYPqUQw1Kg8EIUOTFeCOMolnafeGC4DmHHEQzN/jbIdlCaIyBBC4sd48v/e3Cx6+Idz+8al27OnHzHCzD+2NGf+n10i8wJq6+YvOjgHuqWLjQ8G1pATTmdEZ2N++1q9EkY6GCvMn/iUfKZGrmW9ozeXrOr6ZMNKOyiv6wgNUpVf7Y0DERVRcwjdvJKu8UNtSJXD3MVj8Q0G/SyE2wVDf6PYiQHEirlUxCJQpZPAJAqnuqcNQ0QJG3caJZ3nCUNDT8Qp3PqjB5AjKlGYIblwYZuHj2xnDoDWwORVA3jHm25Niv9KTB/Qow/oz7i8
+  template:
+    metadata:
+      name: grafana-db
+      namespace: monitoring

add-ons/kube-prometheus-stack/grafana-oidc-sealedsecret.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: grafana-oidc
+  namespace: monitoring
+spec:
+  encryptedData:
+    client-secret: AgB2ONbIJxJkVYHuH8kTXuMX5UEFOrOw+UMASkYVQeDT9RBlswBTjtAOuQoZ/++H735rWvV2SD/Hq6PI0rX42lIbasElI83Hnc3PtwvUXajHpvvvB+mM9DH6iPwKzP7M+xgCFXib/4xgWCGa+OnmwfcmYKvWfvBClWQ7t3F0z3jQZNQOytFlXlD8hSCK8w38NwNc4E7JT1YVoH5oUPQq1PE7blsNtM9xqfPqQ+vPfRYjp/j3xXxQhqUHEN7oVZyWDH3UKnhbP+Vd4DzyPvdGZbNbFkzrEqB7ogJOzZAgcsDVs/3NyfDChlCE8MLQ7EnEE7puOdMt8gZhel8kVwBwyARZxy9Y+gvXe//qTyIPcvUdROzycI0QLj5Y8R/QH+AywX70Y7oyezaqcpaS7mRjl8duG8Jg52xFL8z2AjwMZo01vh2GkhxhuQ0ys+jJarQYRUk+cE28ZEZTT22KWmmCo/X1zarymgXO/hzUZwZTDy+GaTh30nCyXOwVipKirQhZCPEVVrT0GOyNz8iMgvbcQgOIFsgSF50vwo5yq2zTTPTtcNJUnfDkN/se/HNpyoEKXaNu2dsio3t7KE8VTaa6rBqw7GvZ81k+nR9stQh4a0U5uxC5Req/CHKvVxtFJXn3TZhlzpYN11zvWjpRMF505jOQVbqYa7xyLio17XF3tYuGUVUJK3AQklL7sO+9ilZhJG5GiqhERxexwEjgKO2j3GFkIPjjnk0BxXLrZtiNDCbW9UvtCBto0Q1qdOoyQTg0Dm0s/B4SmEaRl+9Bfd2943aSxnD799TePoIvgWjXeH9NBA==
+  template:
+    metadata:
+      name: grafana-oidc
+      namespace: monitoring

add-ons/kube-prometheus-stack/httproute.yaml

@@ -0,0 +1,21 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: grafana
+  namespace: monitoring
+spec:
+  hostnames:
+    - grafana.whyredfire.dev
+  parentRefs:
+    - kind: Gateway
+      name: traefik-gateway
+      namespace: traefik
+      sectionName: websecure
+  rules:
+    - backendRefs:
+        - name: kube-prometheus-stack-grafana
+          port: 80
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

add-ons/kube-prometheus-stack/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - grafana-admin-sealedsecret.yaml
+  - grafana-db-sealedsecret.yaml
+  - grafana-oidc-sealedsecret.yaml
+  - httproute.yaml

add-ons/loki/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - s3-sealedsecret.yaml

add-ons/loki/s3-sealedsecret.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: loki-s3
+  namespace: monitoring
+spec:
+  encryptedData:
+    LOKI_S3_ACCESS_KEY_ID: AgDIqzolb+yePeOimTq+WJwLyTsPnCJcivE3UmRURoRHklxpNUyJl6Webd0vEhIsa5rLlb/4UQkz+Lv5MCoqFJJ42z1mZWh3okHFRkf3S8cZlOo11KUlitUbz2YsVuWXEascbb0ScyM0iXvp2BeZSOB2cXX8FnOXs7uw1Zy9lKAEsKaT02p0IceXItCzYmQtNFt94Rzeq0Qej3rXHnXP33XgEsq40JvhAmENAEcfUwWKWksDraGNgBVDHntTRIYUbE4UuSF2ON12baNye7IQyPQJ08FtCYEokLLbcgCe58oUjyecEa2Gx9sGqEyRuumru7bpeQ0N4Uu6O0udhlY3Xpk+Oeo5pt0fJc0b82wG0obVo+/htbVvGvx40vAyjRIVsjJJB03/aMvM0qUystor92PnSn7jxZoe7XS8VS/bq9dqvWDe3CohoxBHU1oib7DB4uKGstUHIqgNQJHZYCtmnNYWEQLfZZlam5qks53nMmlB/BIR3Mzeiui59kjv5peP1JEHEduxOIjRKEP2FPNCds92fNTaIfvbrbrfxWUgeMJNMwauG7krTTgMEosMAM2fQzbQSgzcQS3tGzBDXZBTUSqcadrL4onvlthmh9XrHEsu7m3Ig62BJi2dIWcuHB4345hcaR/B2wtpJJIg+Sy9J8Gu/RfbeiKY80lQFDM4IVX2OExmcp3OTIoDmygyTIj91ydz9HPT8Ly+Jz2oYsmojhWxBeySp6Lm3mH+Ng==
+    LOKI_S3_SECRET_ACCESS_KEY: AgDE/X9NIn7vSjjQQjStswNmTw1Xg/40GlyCUTYWMtL3AizRG8h4BI/NftlM3AfmBeoE0guSOB7S9NG318Wa3wzT9/NB3qW/oHB/Mck+jx+mTvso1jsgTilcKgF1L9HoORG+uvlKp35WX1+8PupdBX9GJ6fvOSimzmBDbukhgjBbf3XMj1obrT89nKHUP5PJYNj/6RSeE7X9jjl6Q2IVH9J5WOt044kSDPliyP9iw09E/70RWxE/KJ4fBZEO/oe1diy3if580rdAG/po3txgs7x0NwC9cHR8V6KehdoJRjzq0a1mgabrn6bbVtfWHzIiKIX47eyjmyPqYMrBG3mXu1SnvbLlyVjG2u67eK+zO07ii5BMF8CzLtuGcMiqusY4HKWVQMYRQRk+A/pTEWWTyAcuwhBfhdF6bVttIQiTia4Uo6qJ2ddwqm5k0Gk46KzfMYcQFn0SAJYfO3s41F0PxfdQ4qEQN+1gb7yw4wZXPCwUYkX+Isc25lslf62R8Aj9NV/rxvFMAFoDNwhLjQZlU2KW1CxfkNpYDaxcLdcXccidHvGrqohPnfDqEEhDDlZIn0xTmM8DcszO3/Pwqz63gX7fBA7zBoRH4ecK3rAFlbeg2vxj+zgtqA/rQ2RUbpmYpI4UiF7xvwKJ7LY5EljPjOiUW7QZ9C158v1jD5sSoWY7AUkfZNX6XUwXq9Cqr7ovA2cxlP7cfZwNb/t3j99KhIEiZVup7PlL8kMJ9tJNqf7iQdM1poq+DpWP+kBjGmwAWk91DMJ8JvSBWePmOf0rJBti
+  template:
+    metadata:
+      name: loki-s3
+      namespace: monitoring

add-ons/postgres/cluster.yaml

@@ -14,6 +14,11 @@ spec:
         name: authentik
         passwordSecret:
           name: authentik-db
+      - ensure: present
+        login: true
+        name: grafana
+        passwordSecret:
+          name: grafana-db
   postgresql:
     # Each app role may only connect to the database of the same name.
     pg_hba:

add-ons/postgres/grafana-database.yaml

@@ -0,0 +1,11 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: grafana
+  namespace: postgres
+spec:
+  cluster:
+    name: postgres
+  databaseReclaimPolicy: retain
+  name: grafana
+  owner: grafana

add-ons/postgres/grafana-db-sealedsecret.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: grafana-db
+  namespace: postgres
+spec:
+  encryptedData:
+    password: AgCVRR7/XrxDHRCzJwfWhC8sPCXuWSMiAQt82cHikAjyDWbTllBVVcZTA5M4U/I1Lz5SVm7rrBNjZuRGYk0q0UXKEJ35HjLAPkAYJ/1/uVy9EdwuzofS0UtzxkcCHwdkgfwHXKRFjCBKP+JoK3bsC2QVJwon9y1FdALv/2lsf+Eed2ncirk3RCyop7R4by9cYvzNt9BU5RYTSO+PtHRF1s2LkmCV+68FQHiVXu7tTw2DfDaTHlflWWIWYNx466+0qZIA0VxZO4wBtzN2q4BxWb/e5pL1Vr3F7HqkAVTAauUoLadc9x8fye2HYfI5MHJSsMhk6OrheqmUaiYZGvWaWHcBTa7HimEAp69sMPv7FSKUME1/g6vh5rQVbX5osmWwKQr68/8Q9CY7uFQfnT7G+C6k0h2dSkbiaczDlOGn2qzrLD1UsKjtaVHxZKzb6FWrbbaCJxiZLm/2Jn0JXZKFpjdk1CYy4xB6ZTNWyqrPkaLqPpMoQV5Zgw44vnhudsTlNWDPYNMdSVJXXsFplxnaxhbzbBjy7JZmO/B3snaxqlZCk57yXW88ZSaNbrRQs1hS+MTNbviy4a6fjcrE5UIBfSFtOle5hPVeLsCOV7h5cdXsZDBYveha2eHC1fnCgK0A7GS65XByJ/wW1J1XDcSz5AIULjLWcLa37W2ftMgYiyOeZsP1vEto3ceKia+vMdY9uyJVmAwN7EiVVGzF0CGbrlTbNRAoMWH/tpKewdkSKLyGHcOp5hWFE+5YxRdSY3maxg+ZskYhJcrWwKK8XIdim+SM
+    username: AgC75g98/7Jys5LOEM2ik0fY+mGxfVisA59k/cJPBdARI1uGApIZ27/BE668Ga0WsDl8aSbybGAZIdEFlPYFs3fGBaJIO3I9ZyOEBH1+KQVto4BkZ+SwqfrUBOw37QFQIbO/Fx3ScONYz9iC5koK857I+G8+ZptRg7ZMRWwFSCynaPu4ju5+3lpPDoaeykvp+wKFt2cPY0r1xcOWGBXbQhTPoCj6UQW1LSClws2pV8I4GIgTW2IvEdx+CT42cTWqZXQO7aCQUxxpgHUXVsB+t6mldQYwn4tEAuPU3IzvCjf7xqGi5XgRGlxfJayUdAOglzSNmaK++YW48YzvWLDSk4/G0oDQzdiHqWldKD4Z3gVnKdhqK07duK7+I2CzusixzoYkjZWATtIC4VWeQKDKNXKwQ513QrHFhjQDYzGQsEdpR5A38Y+S5yNduY+TkivNvx5O24/J/OIR9U1MOqTKsAURl6rAbWGndBlYi2X8/jtFMADo7PO2xZdDSaPlRpM3PvDcD3Iz+7d3LDZUg/L45HJMlp5WmP84ciKL/YfnQgbQWYT4qg+JpvEv5TlDB9aPPt3JDKWJhe6LK1zM59tRsg3YCZNoLp6DdqNnw6TG74ZhaPES/HVLOxt7Efer4vUMPjIOVXZoGmtGlrC5BLN4M7q2ktmj2xUGct68t6NVfPxnhFnHV56QpghLoMZOULLBKaXc+FGHKKst
+  template:
+    metadata:
+      labels:
+        cnpg.io/reload: "true"
+      name: grafana-db
+      namespace: postgres
+    type: kubernetes.io/basic-auth

add-ons/postgres/kustomization.yaml

@@ -4,3 +4,5 @@ resources:
   - cluster.yaml
   - authentik-db-sealedsecret.yaml
   - authentik-database.yaml
+  - grafana-db-sealedsecret.yaml
+  - grafana-database.yaml

argocd-apps/add-ons/alloy.yaml

@@ -0,0 +1,150 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "4"
+  name: alloy
+  namespace: argocd
+spec:
+  destination:
+    namespace: monitoring
+    server: https://kubernetes.default.svc
+  project: default
+  source:
+    chart: alloy
+    helm:
+      values: |
+        alloy:
+          configMap:
+            content: |
+              // Container logs of pods on this node, tailed from /var/log/pods
+              discovery.kubernetes "pods" {
+                role = "pod"
+                selectors {
+                  role  = "pod"
+                  field = "spec.nodeName=" + sys.env("HOSTNAME")
+                }
+              }
+
+              discovery.relabel "pods" {
+                targets = discovery.kubernetes.pods.targets
+
+                rule {
+                  source_labels = ["__meta_kubernetes_namespace"]
+                  target_label  = "namespace"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_name"]
+                  target_label  = "pod"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_container_name"]
+                  target_label  = "container"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"]
+                  regex         = "^;*([^;]+)(;.*)?$"
+                  target_label  = "app"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_node_name"]
+                  target_label  = "node"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+                  separator     = "/"
+                  replacement   = "/var/log/pods/*$1/*.log"
+                  target_label  = "__path__"
+                }
+              }
+
+              local.file_match "pods" {
+                path_targets = discovery.relabel.pods.output
+              }
+
+              loki.source.file "pods" {
+                targets    = local.file_match.pods.targets
+                forward_to = [loki.process.pods.receiver]
+              }
+
+              loki.process "pods" {
+                stage.cri {}
+                forward_to = [loki.write.default.receiver]
+              }
+
+              // Host logs: k3s, containerd, kernel and the rest of systemd
+              loki.relabel "journal" {
+                forward_to = []
+
+                rule {
+                  source_labels = ["__journal__systemd_unit"]
+                  target_label  = "unit"
+                }
+                rule {
+                  source_labels = ["__journal__hostname"]
+                  target_label  = "node"
+                }
+                rule {
+                  source_labels = ["__journal_priority_keyword"]
+                  target_label  = "level"
+                }
+              }
+
+              loki.source.journal "node" {
+                path          = "/var/log/journal"
+                max_age       = "12h"
+                labels        = {job = "node/journal"}
+                relabel_rules = loki.relabel.journal.rules
+                forward_to    = [loki.write.default.receiver]
+              }
+
+              loki.write "default" {
+                endpoint {
+                  url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
+                }
+              }
+          mounts:
+            varlog: true
+            extra:
+              - name: machine-id
+                mountPath: /etc/machine-id
+                readOnly: true
+              - name: storage
+                mountPath: /var/lib/alloy
+          # Tail positions survive pod restarts, so logs are not shipped twice.
+          storagePath: /var/lib/alloy
+          resources:
+            limits:
+              cpu: 500m
+              memory: 512Mi
+            requests:
+              cpu: 50m
+              memory: 128Mi
+        configReloader:
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 50Mi
+        controller:
+          volumes:
+            extra:
+              - name: machine-id
+                hostPath:
+                  path: /etc/machine-id
+                  type: File
+              - name: storage
+                hostPath:
+                  path: /var/lib/alloy
+                  type: DirectoryOrCreate
+    repoURL: https://grafana.github.io/helm-charts
+    targetRevision: 1.13.0
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

argocd-apps/add-ons/kube-prometheus-stack.yaml

@@ -0,0 +1,195 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+  name: kube-prometheus-stack
+  namespace: argocd
+spec:
+  destination:
+    namespace: monitoring
+    server: https://kubernetes.default.svc
+  project: default
+  sources:
+    - chart: kube-prometheus-stack
+      helm:
+        values: |
+          alertmanager:
+            enabled: false
+          # k3s embeds every control-plane component in one process with a shared
+          # metrics registry, so each endpoint also serves the apiserver's series.
+          # Keep those on the apiserver job only.
+          kubelet:
+            serviceMonitor:
+              metricRelabelings:
+                - action: drop
+                  sourceLabels: [__name__, le]
+                  regex: (csi_operations|storage_operation_duration)_seconds_bucket;(0.25|2.5|15|25|120|600)(\.0)?
+                - action: drop
+                  sourceLabels: [__name__]
+                  regex: (apiserver|etcd|kine)_.*
+          # Bound to 0.0.0.0 through /etc/rancher/k3s/config.yaml on the node.
+          kubeControllerManager:
+            endpoints:
+              - 192.168.2.5
+            serviceMonitor:
+              metricRelabelings:
+                - action: drop
+                  sourceLabels: [__name__]
+                  regex: (apiserver|etcd|kine)_.*
+          kubeScheduler:
+            endpoints:
+              - 192.168.2.5
+            serviceMonitor:
+              metricRelabelings:
+                - action: drop
+                  sourceLabels: [__name__]
+                  regex: (apiserver|etcd|kine)_.*
+          kubeProxy:
+            endpoints:
+              - 192.168.2.5
+            serviceMonitor:
+              metricRelabelings:
+                - action: drop
+                  sourceLabels: [__name__]
+                  regex: (apiserver|etcd|kine)_.*
+          # k3s runs on SQLite (kine), there is no etcd to scrape.
+          kubeEtcd:
+            enabled: false
+          prometheusOperator:
+            admissionWebhooks:
+              certManager:
+                enabled: true
+            resources:
+              limits:
+                cpu: 200m
+                memory: 256Mi
+              requests:
+                cpu: 20m
+                memory: 64Mi
+            prometheusConfigReloader:
+              resources:
+                limits:
+                  cpu: 50m
+                  memory: 64Mi
+                requests:
+                  cpu: 10m
+                  memory: 32Mi
+          kube-state-metrics:
+            resources:
+              limits:
+                cpu: 200m
+                memory: 256Mi
+              requests:
+                cpu: 20m
+                memory: 64Mi
+          prometheus-node-exporter:
+            resources:
+              limits:
+                cpu: 200m
+                memory: 128Mi
+              requests:
+                cpu: 20m
+                memory: 32Mi
+          prometheus:
+            prometheusSpec:
+              retention: 90d
+              retentionSize: 45GB
+              # Pick up monitors and rules from every namespace, not just this release.
+              podMonitorSelectorNilUsesHelmValues: false
+              probeSelectorNilUsesHelmValues: false
+              ruleSelectorNilUsesHelmValues: false
+              scrapeConfigSelectorNilUsesHelmValues: false
+              serviceMonitorSelectorNilUsesHelmValues: false
+              # Generous CPU ceiling: WAL replay and compaction burst well above steady state.
+              resources:
+                limits:
+                  cpu: "1"
+                  memory: 3Gi
+                requests:
+                  cpu: 200m
+                  memory: 1Gi
+              storageSpec:
+                volumeClaimTemplate:
+                  spec:
+                    accessModes:
+                      - ReadWriteOnce
+                    resources:
+                      requests:
+                        storage: 50Gi
+          grafana:
+            admin:
+              existingSecret: grafana-admin
+            additionalDataSources:
+              - name: Loki
+                type: loki
+                uid: loki
+                url: http://loki.monitoring.svc.cluster.local:3100
+            envValueFrom:
+              GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
+                secretKeyRef:
+                  name: grafana-oidc
+                  key: client-secret
+              GF_DATABASE_PASSWORD:
+                secretKeyRef:
+                  name: grafana-db
+                  key: password
+            grafana.ini:
+              database:
+                type: postgres
+                host: postgres-rw.postgres.svc.cluster.local:5432
+                name: grafana
+                user: grafana
+                ssl_mode: require
+              server:
+                root_url: https://grafana.whyredfire.dev
+              auth:
+                # Break-glass admin login stays reachable at /login?disableAutoLogin=true
+                oauth_auto_login: true
+                signout_redirect_url: https://auth.whyredfire.dev/application/o/grafana/end-session/
+              auth.generic_oauth:
+                enabled: true
+                name: Authentik
+                client_id: grafana
+                scopes: openid profile email
+                auth_url: https://auth.whyredfire.dev/application/o/authorize/
+                token_url: https://auth.whyredfire.dev/application/o/token/
+                api_url: https://auth.whyredfire.dev/application/o/userinfo/
+                use_pkce: true
+                login_attribute_path: preferred_username
+                role_attribute_path: contains(groups, 'admins') && 'Admin'
+                role_attribute_strict: true
+            # State lives in Postgres; the pod itself is disposable.
+            persistence:
+              enabled: false
+            resources:
+              limits:
+                cpu: 500m
+                memory: 512Mi
+              requests:
+                cpu: 50m
+                memory: 256Mi
+            sidecar:
+              dashboards:
+                searchNamespace: ALL
+              resources:
+                limits:
+                  cpu: 100m
+                  memory: 256Mi
+                requests:
+                  cpu: 10m
+                  memory: 64Mi
+            testFramework:
+              enabled: false
+      repoURL: https://prometheus-community.github.io/helm-charts
+      targetRevision: 91.8.2
+    - path: add-ons/kube-prometheus-stack
+      repoURL: https://gitea.whyredfire.dev/whyredfire/gitops.git
+      targetRevision: main
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

argocd-apps/add-ons/kustomization.yaml

@@ -1,10 +1,13 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - alloy.yaml
   - argocd-dashboard.yaml
   - authentik.yaml
   - cert-manager.yaml
   - cnpg.yaml
+  - kube-prometheus-stack.yaml
+  - loki.yaml
   - postgres.yaml
   - sealed-secrets.yaml
   - traefik.yaml

argocd-apps/add-ons/loki.yaml

@@ -0,0 +1,124 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "4"
+  name: loki
+  namespace: argocd
+spec:
+  destination:
+    namespace: monitoring
+    server: https://kubernetes.default.svc
+  project: default
+  sources:
+    - chart: loki
+      helm:
+        values: |
+          deploymentMode: Monolithic
+          loki:
+            auth_enabled: false
+            commonConfig:
+              replication_factor: 1
+            schemaConfig:
+              configs:
+                - from: "2026-09-30"
+                  store: tsdb
+                  object_store: s3
+                  schema: v13
+                  index:
+                    prefix: loki_index_
+                    period: 24h
+            storage:
+              type: s3
+              bucketNames:
+                chunks: loki
+                ruler: loki
+              s3:
+                endpoint: http://garage.garage.svc.cluster.local:3900
+                region: garage
+                accessKeyId: ${LOKI_S3_ACCESS_KEY_ID}
+                secretAccessKey: ${LOKI_S3_SECRET_ACCESS_KEY}
+                s3ForcePathStyle: true
+                insecure: true
+            ingester:
+              chunk_encoding: zstd
+            compactor:
+              retention_enabled: true
+              delete_request_store: s3
+            limits_config:
+              retention_period: 2160h
+          singleBinary:
+            replicas: 1
+            extraEnvFrom:
+              - secretRef:
+                  name: loki-s3
+            persistence:
+              size: 10Gi
+            resources:
+              limits:
+                cpu: "1"
+                memory: 2Gi
+              requests:
+                cpu: 100m
+                memory: 512Mi
+          # Memory limits sit ~20% above memcached's allocation for connection overhead.
+          chunksCache:
+            allocatedMemory: 512
+            resources:
+              limits:
+                cpu: 500m
+                memory: 614Mi
+              requests:
+                cpu: 50m
+                memory: 614Mi
+          resultsCache:
+            allocatedMemory: 128
+            resources:
+              limits:
+                cpu: 500m
+                memory: 154Mi
+              requests:
+                cpu: 50m
+                memory: 154Mi
+          memcachedExporter:
+            resources:
+              limits:
+                cpu: 100m
+                memory: 64Mi
+              requests:
+                cpu: 10m
+                memory: 16Mi
+          # Loki's rules sidecar
+          sidecar:
+            resources:
+              limits:
+                cpu: 100m
+                memory: 256Mi
+              requests:
+                cpu: 10m
+                memory: 64Mi
+          # Grafana and Alloy talk to Loki directly, there is a single tenant.
+          gateway:
+            enabled: false
+          lokiCanary:
+            enabled: false
+          test:
+            enabled: false
+          backend:
+            replicas: 0
+          read:
+            replicas: 0
+          write:
+            replicas: 0
+      repoURL: https://grafana-community.github.io/helm-charts
+      targetRevision: 18.13.7
+    - path: add-ons/loki
+      repoURL: https://gitea.whyredfire.dev/whyredfire/gitops.git
+      targetRevision: main
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true