From 576243b33e0e94bd99953073d47f7b02f4f0f61c Mon Sep 17 00:00:00 2001 From: Karan Parashar Date: Wed, 30 Sep 2026 00:53:54 +0530 Subject: [PATCH] feat(add-ons): Add observability stack with kube-prometheus-stack, loki and alloy --- add-ons/authentik/blueprints.yaml | 30 +++ add-ons/authentik/sealed-secret.yaml | 1 + .../grafana-admin-sealedsecret.yaml | 14 ++ .../grafana-db-sealedsecret.yaml | 13 ++ .../grafana-oidc-sealedsecret.yaml | 13 ++ add-ons/kube-prometheus-stack/httproute.yaml | 21 ++ .../kube-prometheus-stack/kustomization.yaml | 7 + add-ons/loki/kustomization.yaml | 4 + add-ons/loki/s3-sealedsecret.yaml | 14 ++ add-ons/postgres/cluster.yaml | 5 + add-ons/postgres/grafana-database.yaml | 11 + add-ons/postgres/grafana-db-sealedsecret.yaml | 17 ++ add-ons/postgres/kustomization.yaml | 2 + argocd-apps/add-ons/alloy.yaml | 150 ++++++++++++++ .../add-ons/kube-prometheus-stack.yaml | 195 ++++++++++++++++++ argocd-apps/add-ons/kustomization.yaml | 3 + argocd-apps/add-ons/loki.yaml | 124 +++++++++++ 17 files changed, 624 insertions(+) create mode 100644 add-ons/kube-prometheus-stack/grafana-admin-sealedsecret.yaml create mode 100644 add-ons/kube-prometheus-stack/grafana-db-sealedsecret.yaml create mode 100644 add-ons/kube-prometheus-stack/grafana-oidc-sealedsecret.yaml create mode 100644 add-ons/kube-prometheus-stack/httproute.yaml create mode 100644 add-ons/kube-prometheus-stack/kustomization.yaml create mode 100644 add-ons/loki/kustomization.yaml create mode 100644 add-ons/loki/s3-sealedsecret.yaml create mode 100644 add-ons/postgres/grafana-database.yaml create mode 100644 add-ons/postgres/grafana-db-sealedsecret.yaml create mode 100644 argocd-apps/add-ons/alloy.yaml create mode 100644 argocd-apps/add-ons/kube-prometheus-stack.yaml create mode 100644 argocd-apps/add-ons/loki.yaml diff --git a/add-ons/authentik/blueprints.yaml b/add-ons/authentik/blueprints.yaml index 356b24f..9f7524b 100644 --- a/add-ons/authentik/blueprints.yaml +++ b/add-ons/authentik/blueprints.yaml @@ -67,3 +67,33 @@ data: domain: authentik-default attrs: flow_authentication: !KeyOf google-flow + grafana.yaml: | + version: 1 + metadata: + name: Grafana + entries: + - model: authentik_providers_oauth2.oauth2provider + id: grafana-provider + identifiers: + name: grafana + attrs: + authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]] + invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]] + client_type: confidential + client_id: grafana + client_secret: !Env GRAFANA_CLIENT_SECRET + redirect_uris: + - matching_mode: strict + url: https://grafana.whyredfire.dev/login/generic_oauth + signing_key: !Find [authentik_crypto.certificatekeypair, [name, authentik Self-signed Certificate]] + property_mappings: + - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]] + - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]] + - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-profile]] + - model: authentik_core.application + identifiers: + slug: grafana + attrs: + name: Grafana + provider: !KeyOf grafana-provider + meta_launch_url: https://grafana.whyredfire.dev diff --git a/add-ons/authentik/sealed-secret.yaml b/add-ons/authentik/sealed-secret.yaml index fb66992..0eda694 100644 --- a/add-ons/authentik/sealed-secret.yaml +++ b/add-ons/authentik/sealed-secret.yaml @@ -11,6 +11,7 @@ spec: AUTHENTIK_SECRET_KEY: AgAHHDs8mbJRc45DFgap2VUZn/F+m59hm9u2lebuJv70HSBxLLsHEn8nXLGUO3b2ScyPt1Qi2wbmRDENTjOHOYHtAQAjwV25r6RUiZ5TosRH2KhGx5N6gceorYQyk18l2CkKv4ObXjeGLJA0EqMarmFWiT24SFZdtIrOvWygLxg4XGful2h1Fw5uXK9PiUHgptrp/G5M5Htv0bvtD6m/FrgOlK6C6tK1D4vLL3VLX3/SceqNWVDIJmJao5JKO4IZTQxN8TN5k9K2CGrtOUhjCn4vLbYQKTPhiiidoQaipQ9FR+4KJMO0y8FbIy+3q7C/UZPyOTOu/2zI4t5h4TJY1+477A80GTLpE0aCBDpmFwgZY728wbTW4NyfIm15u0Um7OXhnJkXC1EqDbvlK4HYHiSDUjGChvlfMEY+D1datDlF1fEuIaeXtTqowpz3uy9qIEa0f1emS2J8TjIrKH3oW3mfmrJKSiySnCX1704ZPlwOJfnF5Z2vM6Oqo/mWHa6PSwPPRxSxEKVtSqD4KynzrUmx3GkrUs+7tntepA3yQoiDjyUyE71FwhJXO4sE3XcRGDUf4rB8CFO4+NgTbBAqFzNKYG32GIDhGjbMxBAJQMNXZhVcKceiaXf7G8LuaOEw39BkdtiFCdK3uXl9xaQPuStRsOYYQR2Dt0vt3AvDGjAW+McauCbYUD3rcAtVMo+rxEbejrotojWnb1OpGX0JqD5gHC68qlqY6BYkDbWpYojoKWJHuHlYxHIm8k7ghSq6npw94R526bxripNtcNq4Lh1NeM55LKruOEGr7n0dFcyAmQ== GOOGLE_CLIENT_ID: AgCV2Rwe/bHbkQ3nkkIs4EFTbsEqQWRaWlXa/fHzkDyNKpd0CTOgqVb6Rd2KYyvoT9iDR0xt/8reLPd9e7B3G3V1oVFV7G1XiErBmzvxr6591+tXzo8JLVpIy0363R87idi2xIKGVii+G5aJvxxhua8IRsNfi4SRtoqpDzjgNbFtm5UDhhlCzcRKkm0zHgLw83d4Uwszu9X9II2A4FuMPD6KvsTFrgHevR1v3OerKl5kvTlUx0O3XEg5mnNUVk46J+D9wfg58bIlgUhMV8NtfKa8I5zmRKhCJySks1zlCgIudCLX7wPYupFgt0nrZjJGx7a546ntPdMVE4IJ+VhlQ94gjdWsVFm07espaQLph9T4CZDKxb6TwyNswhfFxH1Z9auyFf3Cfk2wx5GChNOL0mZ8Zrzcw008c8IUShF8je8JZ60teBffbBNao7XN6JM6a4ac5aKnNwFch9IJUU1qkvcJaZk5hAkeGO9uHrICb6lkr5Rj7t4h9l4mvgfIKXP5YCCKZ+yqrSI8Jpkch/cB0Sphgp7m7QXgGwmWRgdKEk1rP7ioIfoF66mLsfgvOkc+1ySvPgW/rCLbHwzMqJn1Fkk/afbRoWp/guFj1cOY9MrEQIn8ftugeV/0xWiL+y5RyJm7QZc1z5fSj/NTrVCZV0LhBHttPZzoGGWEqjLYQnyJ8p38lK7EU6Pmk4WrEMj7mmdYHEijyPhab+b7guOc+ULbPT7k7sBHXbKrTTUJa09jIqSS4jl2PNrzPCPDIvRpG/cMgNklhNPfvpDQcLdVRA7bsf7navZvDsQ= GOOGLE_CLIENT_SECRET: AgDNFpqV5LGekTMvOBtPE9c6ZVPtT6wN+nCFRhZnvBKIz4I7U9jzisEN7yhc4ud22XJXpS5oR8hN4H+V6Js/bCZMDJefYcdstmHku14kUvoLRUtwPV9tdjiihmb57VLTnT1s1OzbnOqZPwXoCAmGAPLDgZrghBi0id21GgjU41jCHDJCcH1j9RHhE20JwsVG2uz9Tcp6MVYh94aq53V4//bvFXz4xQgVlN8c1jJ/W4rrNM9vmYdMwKyB5509oVWJigowsRlji0FXurR8ovz1AXpMuGj6hUGxLp0uHsqz7FV/0lsd4B2BR9cVRChAR8zDCiiKb4TH2Kmn676/Z+gtJ8vcOQg/WDT1ENzOr6dyndCLVxJvJuvl1EqinzhSqJjUy11HxqULApKKbX1iRJIAIGb6dpeKVoHDKg5xtcWGwaD6epG/tz20agkWBMCFNq5aS80npz28W5GeOQ8aF4X/rpjuEHVLrN+n0pgyS4wjwniB6UJMYRIYk45r0ICUtqyFjOiNzpU70NsPmEjDyItW1EVo+6tb0lRmOI5apm/9yayZXjFISwTAFPdBMrQL+ij4AGw4HMWMmN0aqDO+Kd1c9Jg6wUKluN2Lb6CaxehWeSJiz2wTEhEz8KBkInfezgSekNkw4wjJiBEZ4/M3G6Y/C61ekuVFarRc87cY+S3SomVZDGQcPysfQlieMVpHlfXq/CSDgHhZV3Q+S/wqpy5ZcfsyaU+v4PmvyEc12MZxQ1RGngVJow== + GRAFANA_CLIENT_SECRET: AgBSE0QjGwtnyAjO/cy/TBKg+fuyKMSRgeQM/i7tPi9QcLpDjEHquGKmdBHUaeSaon3LRH3CPYQJ61T4d4xsp7EXzCM47MhO/rJ+6Xx+dGiIg071Ijj1qOBYszMNPXLZCst/9EfDEBRXQYmtr8X5UfcyXv8RHewvuqxcwy8qzsPsP5G/CpchGWlZRV3x+R+yCjVmTP7VuatPAltvGuAHsAv3Gh1aEXMJ1F3SsW/SP025V/bMiz1wKWe7/bkqJnXjLkNXWI/lpwzYWQl4/9v1Oiiw+1osH6fHNzXM6RA0ei1/FmlRwS9EqnUnl9hGE76GSllhwU9L6K9boJrKRnZIP6xBOJfrqimZciutTfiu6urghyimVACafQLWkeEB8m/zX1CSxVQ5M95n/dtdlBsWofMVyEtoeTip1HXZF1iee4yMv2yEg8yNSWmnb87fSYIMwpMiv5dYQTLg2WslnEe0DxcnyH+O0/MsmWHFz2f/Z4J6h+kWwMEv1pt8l+xkgbUthGGDx+HIcU9SZ/jyZjnzTt4i4BIbejAoXzMmm6q4zE0BD+9AbAQvEDOg6DW1TIu2mupoKVlqYljNE/9zVOAIBcFL0cJ4HNnW13+ufS4KoptZE3Rfv1tpU6g2MZ4y+QBCOczd+5HgXMtv3P9OweZg/l/LRdbFyH3OZviW/4suJFNbWZNgGUlqT/xyk/C5rn7uGSjSinp0lwkAqfxx7To/KP0jE7hJkVOH1vh+7COpFABJEx/cLAXAaIaVJsio38p5i4s+SMEq85Ew3uI46YKD8ljY7AQN3k66U7xQkLgjAT170Q== template: metadata: name: authentik-secrets diff --git a/add-ons/kube-prometheus-stack/grafana-admin-sealedsecret.yaml b/add-ons/kube-prometheus-stack/grafana-admin-sealedsecret.yaml new file mode 100644 index 0000000..9909499 --- /dev/null +++ b/add-ons/kube-prometheus-stack/grafana-admin-sealedsecret.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: grafana-admin + namespace: monitoring +spec: + encryptedData: + admin-password: AgAvoctRn9pH4kx3i0Q75rwFQYuAavZ7/1sgIHzmuOhzR8qPJVLeMwgmbZHz/dcNwD59tKpk1WmzrazUSdTAgkAKCs+CsxXxmxYoaDx+tUEMxHzpUKtW3On6a9YMzhp1e7iTewSMQVW+JZxnubxgT8Ws5N6iDlb/SVKGqrGMMk0436L4qRrXJvtmFRbcjn0h6dZHjuBNDBsGXh5TJCk07IIDYkaCpkZoJvH4Brf3Zla0jgRI3FQ2qFygButR1HLZQ6n7CPq9rhKhzSt3NWktV4zuU3GnZXgk8h9TML09UVEfv8W3NkKmIoXb7DewMhTbtcGw61eV6oCAERv0G2lVIBUQ6Zrfj942QB3KKvXLWzKNlCOMgV/JJUtEFw+lnsB9CLJzEl7eSodsIIFQIaby4iEozjaB2QtWtrROmR7mQDEqk1N0AUHx8oiuJFtJDeTn/0dzswPT+SRWlYoGr54uxHo0nFwkAZyEQehdrsd0Tor9BlXYt510fTSC4VZKHjFbDggyLPn+rmNrShQKB8yfeouovR/SaMa9LG0GpGR51dmDmtH64pN8ukCvMyjXrttD8qiArj+ysywTA2Fge61m1bw4dhen5v7HUrfZW5TxualndhhcUA1VWB9rljZvCqrzeYAwbdj36uTC37mqqhwa36SNKG/AHjKBJ0968TQRgf+DvP+xyIeqFk/tBA4FIfYkDM1jHrSlydWQ7vPca6pbXFtTVA6xUYpqraeCdcIK1n9mxFBG513W + admin-user: AgAzRobKF4UVBc21QSAN42jmMSULMTO5F6wkOYIBqgUMyVAdSD1yW+9KP8s7Bu0+1efiqeAJGtB74IJ/8Py7szgHVv1jqHpKz9hqd8A+nJjAhAXvCaERMzM5KZu1ArAmRX6X6H2OdveOgii8la5mrOBDOXVFiG7AMYzjlP+Ia5H/5HBHsEAEKUUNZF1DCznOS448dGftyKK8XEDkCca9B62rFyV+fjEhnTVkvqlg++wgCv/YqHOWzT1uSXLtOBQzoTiYJM6D1oC4n8+xuQx6o7Z/sy6K1Cc5Ypx0ENPpuPwrIUgfpWt84XgKKM94tzN3szFXrlZe2uGkh5gNTEiSnjRa5azsGU/pp8uNV01FUwCORKbWtvyHm+mLM3V1qMhlQpenF92pOVlsMspvxMMaE9iWz5CaQl7QWEH25rjrlI1FGIQx5tQwZrOD2u7qRZ1xv1rsG/GfvKN3tTj7HkhbHL+OIkJHxEx4Doc6yoW57M8sV3b/18TuUWuRazBxn9RkNhcEd/CP/ZhKW13OhMm/U1fuzPDGqgWHoz+jVM7XCb9/p+X0vxUAt2ZiOAc1n8tGDglxkSYhoZX6agn6fviSaEWskalqpisiSpvpzQra/HxoyWqNG4+ovHiRM1yCdSxdsZoziVuVdnjP2wDV525nuU+38uDxMgAxWEigEFr5sOb5LFKKC+nnSwxaQ/+vk4NPJGqTuu9tGA== + template: + metadata: + name: grafana-admin + namespace: monitoring diff --git a/add-ons/kube-prometheus-stack/grafana-db-sealedsecret.yaml b/add-ons/kube-prometheus-stack/grafana-db-sealedsecret.yaml new file mode 100644 index 0000000..d95116d --- /dev/null +++ b/add-ons/kube-prometheus-stack/grafana-db-sealedsecret.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: grafana-db + namespace: monitoring +spec: + encryptedData: + password: AgCHwt3pHs+dvK/Dg3/PlRvDnC+LjOg42GXKVl61+wXcJdeIc2dtAHLnu1/OOyOOuIYFvovgS02GVH2RmWQ5kkfZ/lD7XRTJxmYBsBiiYqU5rMENy+EJ4bBLcDw7AoPZBowR3y+nl49qHSuCqPWvLwpQ4hh9OXtB0XpEOD8Vqdpr7t3reEMA0T3xd8TrdriNo2sHro5v8o8qMJvZ/6QGmqZWr0yOPMuNSfm4cTaJLCL/MsYQRsUuJ4+1+LLoldrNjSalYqhamQvA6UsVOnuH0Ajp8rj7lnbyf/PbUx+cvqT+4IkDxDnDffdN66Q5s+oND+fMEM64QhFBP7KPrw1rWQMTC+awe5mjf37h0J1JFtnty4p4gJ6eXjc2wQVleuxDzZP9m26bp0Lf00cMTxxWDSHeZsKg4aQ0aqsSKYyXg2i2Nu/UEX/NBrNGTCFNk8+QIGowYr8KJkgC/WmaUNBLxpiNW6YIPwYPqUQw1Kg8EIUOTFeCOMolnafeGC4DmHHEQzN/jbIdlCaIyBBC4sd48v/e3Cx6+Idz+8al27OnHzHCzD+2NGf+n10i8wJq6+YvOjgHuqWLjQ8G1pATTmdEZ2N++1q9EkY6GCvMn/iUfKZGrmW9ozeXrOr6ZMNKOyiv6wgNUpVf7Y0DERVRcwjdvJKu8UNtSJXD3MVj8Q0G/SyE2wVDf6PYiQHEirlUxCJQpZPAJAqnuqcNQ0QJG3caJZ3nCUNDT8Qp3PqjB5AjKlGYIblwYZuHj2xnDoDWwORVA3jHm25Niv9KTB/Qow/oz7i8 + template: + metadata: + name: grafana-db + namespace: monitoring diff --git a/add-ons/kube-prometheus-stack/grafana-oidc-sealedsecret.yaml b/add-ons/kube-prometheus-stack/grafana-oidc-sealedsecret.yaml new file mode 100644 index 0000000..a1d27dc --- /dev/null +++ b/add-ons/kube-prometheus-stack/grafana-oidc-sealedsecret.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: grafana-oidc + namespace: monitoring +spec: + encryptedData: + client-secret: AgB2ONbIJxJkVYHuH8kTXuMX5UEFOrOw+UMASkYVQeDT9RBlswBTjtAOuQoZ/++H735rWvV2SD/Hq6PI0rX42lIbasElI83Hnc3PtwvUXajHpvvvB+mM9DH6iPwKzP7M+xgCFXib/4xgWCGa+OnmwfcmYKvWfvBClWQ7t3F0z3jQZNQOytFlXlD8hSCK8w38NwNc4E7JT1YVoH5oUPQq1PE7blsNtM9xqfPqQ+vPfRYjp/j3xXxQhqUHEN7oVZyWDH3UKnhbP+Vd4DzyPvdGZbNbFkzrEqB7ogJOzZAgcsDVs/3NyfDChlCE8MLQ7EnEE7puOdMt8gZhel8kVwBwyARZxy9Y+gvXe//qTyIPcvUdROzycI0QLj5Y8R/QH+AywX70Y7oyezaqcpaS7mRjl8duG8Jg52xFL8z2AjwMZo01vh2GkhxhuQ0ys+jJarQYRUk+cE28ZEZTT22KWmmCo/X1zarymgXO/hzUZwZTDy+GaTh30nCyXOwVipKirQhZCPEVVrT0GOyNz8iMgvbcQgOIFsgSF50vwo5yq2zTTPTtcNJUnfDkN/se/HNpyoEKXaNu2dsio3t7KE8VTaa6rBqw7GvZ81k+nR9stQh4a0U5uxC5Req/CHKvVxtFJXn3TZhlzpYN11zvWjpRMF505jOQVbqYa7xyLio17XF3tYuGUVUJK3AQklL7sO+9ilZhJG5GiqhERxexwEjgKO2j3GFkIPjjnk0BxXLrZtiNDCbW9UvtCBto0Q1qdOoyQTg0Dm0s/B4SmEaRl+9Bfd2943aSxnD799TePoIvgWjXeH9NBA== + template: + metadata: + name: grafana-oidc + namespace: monitoring diff --git a/add-ons/kube-prometheus-stack/httproute.yaml b/add-ons/kube-prometheus-stack/httproute.yaml new file mode 100644 index 0000000..91558dc --- /dev/null +++ b/add-ons/kube-prometheus-stack/httproute.yaml @@ -0,0 +1,21 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: grafana + namespace: monitoring +spec: + hostnames: + - grafana.whyredfire.dev + parentRefs: + - kind: Gateway + name: traefik-gateway + namespace: traefik + sectionName: websecure + rules: + - backendRefs: + - name: kube-prometheus-stack-grafana + port: 80 + matches: + - path: + type: PathPrefix + value: / diff --git a/add-ons/kube-prometheus-stack/kustomization.yaml b/add-ons/kube-prometheus-stack/kustomization.yaml new file mode 100644 index 0000000..6ee7174 --- /dev/null +++ b/add-ons/kube-prometheus-stack/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - grafana-admin-sealedsecret.yaml + - grafana-db-sealedsecret.yaml + - grafana-oidc-sealedsecret.yaml + - httproute.yaml diff --git a/add-ons/loki/kustomization.yaml b/add-ons/loki/kustomization.yaml new file mode 100644 index 0000000..553468c --- /dev/null +++ b/add-ons/loki/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - s3-sealedsecret.yaml diff --git a/add-ons/loki/s3-sealedsecret.yaml b/add-ons/loki/s3-sealedsecret.yaml new file mode 100644 index 0000000..3fc5fa6 --- /dev/null +++ b/add-ons/loki/s3-sealedsecret.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: loki-s3 + namespace: monitoring +spec: + encryptedData: + LOKI_S3_ACCESS_KEY_ID: AgDIqzolb+yePeOimTq+WJwLyTsPnCJcivE3UmRURoRHklxpNUyJl6Webd0vEhIsa5rLlb/4UQkz+Lv5MCoqFJJ42z1mZWh3okHFRkf3S8cZlOo11KUlitUbz2YsVuWXEascbb0ScyM0iXvp2BeZSOB2cXX8FnOXs7uw1Zy9lKAEsKaT02p0IceXItCzYmQtNFt94Rzeq0Qej3rXHnXP33XgEsq40JvhAmENAEcfUwWKWksDraGNgBVDHntTRIYUbE4UuSF2ON12baNye7IQyPQJ08FtCYEokLLbcgCe58oUjyecEa2Gx9sGqEyRuumru7bpeQ0N4Uu6O0udhlY3Xpk+Oeo5pt0fJc0b82wG0obVo+/htbVvGvx40vAyjRIVsjJJB03/aMvM0qUystor92PnSn7jxZoe7XS8VS/bq9dqvWDe3CohoxBHU1oib7DB4uKGstUHIqgNQJHZYCtmnNYWEQLfZZlam5qks53nMmlB/BIR3Mzeiui59kjv5peP1JEHEduxOIjRKEP2FPNCds92fNTaIfvbrbrfxWUgeMJNMwauG7krTTgMEosMAM2fQzbQSgzcQS3tGzBDXZBTUSqcadrL4onvlthmh9XrHEsu7m3Ig62BJi2dIWcuHB4345hcaR/B2wtpJJIg+Sy9J8Gu/RfbeiKY80lQFDM4IVX2OExmcp3OTIoDmygyTIj91ydz9HPT8Ly+Jz2oYsmojhWxBeySp6Lm3mH+Ng== + LOKI_S3_SECRET_ACCESS_KEY: AgDE/X9NIn7vSjjQQjStswNmTw1Xg/40GlyCUTYWMtL3AizRG8h4BI/NftlM3AfmBeoE0guSOB7S9NG318Wa3wzT9/NB3qW/oHB/Mck+jx+mTvso1jsgTilcKgF1L9HoORG+uvlKp35WX1+8PupdBX9GJ6fvOSimzmBDbukhgjBbf3XMj1obrT89nKHUP5PJYNj/6RSeE7X9jjl6Q2IVH9J5WOt044kSDPliyP9iw09E/70RWxE/KJ4fBZEO/oe1diy3if580rdAG/po3txgs7x0NwC9cHR8V6KehdoJRjzq0a1mgabrn6bbVtfWHzIiKIX47eyjmyPqYMrBG3mXu1SnvbLlyVjG2u67eK+zO07ii5BMF8CzLtuGcMiqusY4HKWVQMYRQRk+A/pTEWWTyAcuwhBfhdF6bVttIQiTia4Uo6qJ2ddwqm5k0Gk46KzfMYcQFn0SAJYfO3s41F0PxfdQ4qEQN+1gb7yw4wZXPCwUYkX+Isc25lslf62R8Aj9NV/rxvFMAFoDNwhLjQZlU2KW1CxfkNpYDaxcLdcXccidHvGrqohPnfDqEEhDDlZIn0xTmM8DcszO3/Pwqz63gX7fBA7zBoRH4ecK3rAFlbeg2vxj+zgtqA/rQ2RUbpmYpI4UiF7xvwKJ7LY5EljPjOiUW7QZ9C158v1jD5sSoWY7AUkfZNX6XUwXq9Cqr7ovA2cxlP7cfZwNb/t3j99KhIEiZVup7PlL8kMJ9tJNqf7iQdM1poq+DpWP+kBjGmwAWk91DMJ8JvSBWePmOf0rJBti + template: + metadata: + name: loki-s3 + namespace: monitoring diff --git a/add-ons/postgres/cluster.yaml b/add-ons/postgres/cluster.yaml index 9b916ea..d40b4ea 100644 --- a/add-ons/postgres/cluster.yaml +++ b/add-ons/postgres/cluster.yaml @@ -14,6 +14,11 @@ spec: name: authentik passwordSecret: name: authentik-db + - ensure: present + login: true + name: grafana + passwordSecret: + name: grafana-db postgresql: # Each app role may only connect to the database of the same name. pg_hba: diff --git a/add-ons/postgres/grafana-database.yaml b/add-ons/postgres/grafana-database.yaml new file mode 100644 index 0000000..cab72c6 --- /dev/null +++ b/add-ons/postgres/grafana-database.yaml @@ -0,0 +1,11 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: grafana + namespace: postgres +spec: + cluster: + name: postgres + databaseReclaimPolicy: retain + name: grafana + owner: grafana diff --git a/add-ons/postgres/grafana-db-sealedsecret.yaml b/add-ons/postgres/grafana-db-sealedsecret.yaml new file mode 100644 index 0000000..1b57032 --- /dev/null +++ b/add-ons/postgres/grafana-db-sealedsecret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: grafana-db + namespace: postgres +spec: + encryptedData: + password: AgCVRR7/XrxDHRCzJwfWhC8sPCXuWSMiAQt82cHikAjyDWbTllBVVcZTA5M4U/I1Lz5SVm7rrBNjZuRGYk0q0UXKEJ35HjLAPkAYJ/1/uVy9EdwuzofS0UtzxkcCHwdkgfwHXKRFjCBKP+JoK3bsC2QVJwon9y1FdALv/2lsf+Eed2ncirk3RCyop7R4by9cYvzNt9BU5RYTSO+PtHRF1s2LkmCV+68FQHiVXu7tTw2DfDaTHlflWWIWYNx466+0qZIA0VxZO4wBtzN2q4BxWb/e5pL1Vr3F7HqkAVTAauUoLadc9x8fye2HYfI5MHJSsMhk6OrheqmUaiYZGvWaWHcBTa7HimEAp69sMPv7FSKUME1/g6vh5rQVbX5osmWwKQr68/8Q9CY7uFQfnT7G+C6k0h2dSkbiaczDlOGn2qzrLD1UsKjtaVHxZKzb6FWrbbaCJxiZLm/2Jn0JXZKFpjdk1CYy4xB6ZTNWyqrPkaLqPpMoQV5Zgw44vnhudsTlNWDPYNMdSVJXXsFplxnaxhbzbBjy7JZmO/B3snaxqlZCk57yXW88ZSaNbrRQs1hS+MTNbviy4a6fjcrE5UIBfSFtOle5hPVeLsCOV7h5cdXsZDBYveha2eHC1fnCgK0A7GS65XByJ/wW1J1XDcSz5AIULjLWcLa37W2ftMgYiyOeZsP1vEto3ceKia+vMdY9uyJVmAwN7EiVVGzF0CGbrlTbNRAoMWH/tpKewdkSKLyGHcOp5hWFE+5YxRdSY3maxg+ZskYhJcrWwKK8XIdim+SM + username: AgC75g98/7Jys5LOEM2ik0fY+mGxfVisA59k/cJPBdARI1uGApIZ27/BE668Ga0WsDl8aSbybGAZIdEFlPYFs3fGBaJIO3I9ZyOEBH1+KQVto4BkZ+SwqfrUBOw37QFQIbO/Fx3ScONYz9iC5koK857I+G8+ZptRg7ZMRWwFSCynaPu4ju5+3lpPDoaeykvp+wKFt2cPY0r1xcOWGBXbQhTPoCj6UQW1LSClws2pV8I4GIgTW2IvEdx+CT42cTWqZXQO7aCQUxxpgHUXVsB+t6mldQYwn4tEAuPU3IzvCjf7xqGi5XgRGlxfJayUdAOglzSNmaK++YW48YzvWLDSk4/G0oDQzdiHqWldKD4Z3gVnKdhqK07duK7+I2CzusixzoYkjZWATtIC4VWeQKDKNXKwQ513QrHFhjQDYzGQsEdpR5A38Y+S5yNduY+TkivNvx5O24/J/OIR9U1MOqTKsAURl6rAbWGndBlYi2X8/jtFMADo7PO2xZdDSaPlRpM3PvDcD3Iz+7d3LDZUg/L45HJMlp5WmP84ciKL/YfnQgbQWYT4qg+JpvEv5TlDB9aPPt3JDKWJhe6LK1zM59tRsg3YCZNoLp6DdqNnw6TG74ZhaPES/HVLOxt7Efer4vUMPjIOVXZoGmtGlrC5BLN4M7q2ktmj2xUGct68t6NVfPxnhFnHV56QpghLoMZOULLBKaXc+FGHKKst + template: + metadata: + labels: + cnpg.io/reload: "true" + name: grafana-db + namespace: postgres + type: kubernetes.io/basic-auth diff --git a/add-ons/postgres/kustomization.yaml b/add-ons/postgres/kustomization.yaml index 36da0c9..84bca99 100644 --- a/add-ons/postgres/kustomization.yaml +++ b/add-ons/postgres/kustomization.yaml @@ -4,3 +4,5 @@ resources: - cluster.yaml - authentik-db-sealedsecret.yaml - authentik-database.yaml + - grafana-db-sealedsecret.yaml + - grafana-database.yaml diff --git a/argocd-apps/add-ons/alloy.yaml b/argocd-apps/add-ons/alloy.yaml new file mode 100644 index 0000000..edf09f7 --- /dev/null +++ b/argocd-apps/add-ons/alloy.yaml @@ -0,0 +1,150 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + annotations: + argocd.argoproj.io/sync-wave: "4" + name: alloy + namespace: argocd +spec: + destination: + namespace: monitoring + server: https://kubernetes.default.svc + project: default + source: + chart: alloy + helm: + values: | + alloy: + configMap: + content: | + // Container logs of pods on this node, tailed from /var/log/pods + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + sys.env("HOSTNAME") + } + } + + discovery.relabel "pods" { + targets = discovery.kubernetes.pods.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"] + regex = "^;*([^;]+)(;.*)?$" + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pods" { + path_targets = discovery.relabel.pods.output + } + + loki.source.file "pods" { + targets = local.file_match.pods.targets + forward_to = [loki.process.pods.receiver] + } + + loki.process "pods" { + stage.cri {} + forward_to = [loki.write.default.receiver] + } + + // Host logs: k3s, containerd, kernel and the rest of systemd + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "node" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + + loki.source.journal "node" { + path = "/var/log/journal" + max_age = "12h" + labels = {job = "node/journal"} + relabel_rules = loki.relabel.journal.rules + forward_to = [loki.write.default.receiver] + } + + loki.write "default" { + endpoint { + url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push" + } + } + mounts: + varlog: true + extra: + - name: machine-id + mountPath: /etc/machine-id + readOnly: true + - name: storage + mountPath: /var/lib/alloy + # Tail positions survive pod restarts, so logs are not shipped twice. + storagePath: /var/lib/alloy + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 128Mi + configReloader: + resources: + limits: + cpu: 50m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + controller: + volumes: + extra: + - name: machine-id + hostPath: + path: /etc/machine-id + type: File + - name: storage + hostPath: + path: /var/lib/alloy + type: DirectoryOrCreate + repoURL: https://grafana.github.io/helm-charts + targetRevision: 1.13.0 + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd-apps/add-ons/kube-prometheus-stack.yaml b/argocd-apps/add-ons/kube-prometheus-stack.yaml new file mode 100644 index 0000000..795731e --- /dev/null +++ b/argocd-apps/add-ons/kube-prometheus-stack.yaml @@ -0,0 +1,195 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + annotations: + argocd.argoproj.io/sync-wave: "3" + name: kube-prometheus-stack + namespace: argocd +spec: + destination: + namespace: monitoring + server: https://kubernetes.default.svc + project: default + sources: + - chart: kube-prometheus-stack + helm: + values: | + alertmanager: + enabled: false + # k3s embeds every control-plane component in one process with a shared + # metrics registry, so each endpoint also serves the apiserver's series. + # Keep those on the apiserver job only. + kubelet: + serviceMonitor: + metricRelabelings: + - action: drop + sourceLabels: [__name__, le] + regex: (csi_operations|storage_operation_duration)_seconds_bucket;(0.25|2.5|15|25|120|600)(\.0)? + - action: drop + sourceLabels: [__name__] + regex: (apiserver|etcd|kine)_.* + # Bound to 0.0.0.0 through /etc/rancher/k3s/config.yaml on the node. + kubeControllerManager: + endpoints: + - 192.168.2.5 + serviceMonitor: + metricRelabelings: + - action: drop + sourceLabels: [__name__] + regex: (apiserver|etcd|kine)_.* + kubeScheduler: + endpoints: + - 192.168.2.5 + serviceMonitor: + metricRelabelings: + - action: drop + sourceLabels: [__name__] + regex: (apiserver|etcd|kine)_.* + kubeProxy: + endpoints: + - 192.168.2.5 + serviceMonitor: + metricRelabelings: + - action: drop + sourceLabels: [__name__] + regex: (apiserver|etcd|kine)_.* + # k3s runs on SQLite (kine), there is no etcd to scrape. + kubeEtcd: + enabled: false + prometheusOperator: + admissionWebhooks: + certManager: + enabled: true + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 20m + memory: 64Mi + prometheusConfigReloader: + resources: + limits: + cpu: 50m + memory: 64Mi + requests: + cpu: 10m + memory: 32Mi + kube-state-metrics: + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 20m + memory: 64Mi + prometheus-node-exporter: + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 20m + memory: 32Mi + prometheus: + prometheusSpec: + retention: 90d + retentionSize: 45GB + # Pick up monitors and rules from every namespace, not just this release. + podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + scrapeConfigSelectorNilUsesHelmValues: false + serviceMonitorSelectorNilUsesHelmValues: false + # Generous CPU ceiling: WAL replay and compaction burst well above steady state. + resources: + limits: + cpu: "1" + memory: 3Gi + requests: + cpu: 200m + memory: 1Gi + storageSpec: + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + grafana: + admin: + existingSecret: grafana-admin + additionalDataSources: + - name: Loki + type: loki + uid: loki + url: http://loki.monitoring.svc.cluster.local:3100 + envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-oidc + key: client-secret + GF_DATABASE_PASSWORD: + secretKeyRef: + name: grafana-db + key: password + grafana.ini: + database: + type: postgres + host: postgres-rw.postgres.svc.cluster.local:5432 + name: grafana + user: grafana + ssl_mode: require + server: + root_url: https://grafana.whyredfire.dev + auth: + # Break-glass admin login stays reachable at /login?disableAutoLogin=true + oauth_auto_login: true + signout_redirect_url: https://auth.whyredfire.dev/application/o/grafana/end-session/ + auth.generic_oauth: + enabled: true + name: Authentik + client_id: grafana + scopes: openid profile email + auth_url: https://auth.whyredfire.dev/application/o/authorize/ + token_url: https://auth.whyredfire.dev/application/o/token/ + api_url: https://auth.whyredfire.dev/application/o/userinfo/ + use_pkce: true + login_attribute_path: preferred_username + role_attribute_path: contains(groups, 'admins') && 'Admin' + role_attribute_strict: true + # State lives in Postgres; the pod itself is disposable. + persistence: + enabled: false + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 256Mi + sidecar: + dashboards: + searchNamespace: ALL + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 64Mi + testFramework: + enabled: false + repoURL: https://prometheus-community.github.io/helm-charts + targetRevision: 91.8.2 + - path: add-ons/kube-prometheus-stack + repoURL: https://gitea.whyredfire.dev/whyredfire/gitops.git + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd-apps/add-ons/kustomization.yaml b/argocd-apps/add-ons/kustomization.yaml index 35b27fe..f222c2f 100644 --- a/argocd-apps/add-ons/kustomization.yaml +++ b/argocd-apps/add-ons/kustomization.yaml @@ -1,10 +1,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - alloy.yaml - argocd-dashboard.yaml - authentik.yaml - cert-manager.yaml - cnpg.yaml + - kube-prometheus-stack.yaml + - loki.yaml - postgres.yaml - sealed-secrets.yaml - traefik.yaml diff --git a/argocd-apps/add-ons/loki.yaml b/argocd-apps/add-ons/loki.yaml new file mode 100644 index 0000000..70b81b1 --- /dev/null +++ b/argocd-apps/add-ons/loki.yaml @@ -0,0 +1,124 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + annotations: + argocd.argoproj.io/sync-wave: "4" + name: loki + namespace: argocd +spec: + destination: + namespace: monitoring + server: https://kubernetes.default.svc + project: default + sources: + - chart: loki + helm: + values: | + deploymentMode: Monolithic + loki: + auth_enabled: false + commonConfig: + replication_factor: 1 + schemaConfig: + configs: + - from: "2026-09-01" + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + storage: + type: s3 + bucketNames: + chunks: loki + ruler: loki + s3: + endpoint: http://garage.garage.svc.cluster.local:3900 + region: garage + accessKeyId: ${LOKI_S3_ACCESS_KEY_ID} + secretAccessKey: ${LOKI_S3_SECRET_ACCESS_KEY} + s3ForcePathStyle: true + insecure: true + ingester: + chunk_encoding: zstd + compactor: + retention_enabled: true + delete_request_store: s3 + limits_config: + retention_period: 2160h + singleBinary: + replicas: 1 + extraEnvFrom: + - secretRef: + name: loki-s3 + persistence: + size: 10Gi + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 100m + memory: 512Mi + # Memory limits sit ~20% above memcached's allocation for connection overhead. + chunksCache: + allocatedMemory: 512 + resources: + limits: + cpu: 500m + memory: 614Mi + requests: + cpu: 50m + memory: 614Mi + resultsCache: + allocatedMemory: 128 + resources: + limits: + cpu: 500m + memory: 154Mi + requests: + cpu: 50m + memory: 154Mi + memcachedExporter: + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 10m + memory: 16Mi + # Loki's rules sidecar + sidecar: + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 10m + memory: 64Mi + # Grafana and Alloy talk to Loki directly, there is a single tenant. + gateway: + enabled: false + lokiCanary: + enabled: false + test: + enabled: false + backend: + replicas: 0 + read: + replicas: 0 + write: + replicas: 0 + repoURL: https://grafana-community.github.io/helm-charts + targetRevision: 18.13.7 + - path: add-ons/loki + repoURL: https://gitea.whyredfire.dev/whyredfire/gitops.git + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true