feat(add-ons): Add observability stack with kube-prometheus-stack, loki and alloy

2026-09-30 01:09:34 +05:30
parent 148890fb86
commit 576243b33e
17 changed files with 624 additions and 0 deletions
@@ -0,0 +1,150 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "4"
+  name: alloy
+  namespace: argocd
+spec:
+  destination:
+    namespace: monitoring
+    server: https://kubernetes.default.svc
+  project: default
+  source:
+    chart: alloy
+    helm:
+      values: |
+        alloy:
+          configMap:
+            content: |
+              // Container logs of pods on this node, tailed from /var/log/pods
+              discovery.kubernetes "pods" {
+                role = "pod"
+                selectors {
+                  role  = "pod"
+                  field = "spec.nodeName=" + sys.env("HOSTNAME")
+                }
+              }
+
+              discovery.relabel "pods" {
+                targets = discovery.kubernetes.pods.targets
+
+                rule {
+                  source_labels = ["__meta_kubernetes_namespace"]
+                  target_label  = "namespace"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_name"]
+                  target_label  = "pod"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_container_name"]
+                  target_label  = "container"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"]
+                  regex         = "^;*([^;]+)(;.*)?$"
+                  target_label  = "app"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_node_name"]
+                  target_label  = "node"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+                  separator     = "/"
+                  replacement   = "/var/log/pods/*$1/*.log"
+                  target_label  = "__path__"
+                }
+              }
+
+              local.file_match "pods" {
+                path_targets = discovery.relabel.pods.output
+              }
+
+              loki.source.file "pods" {
+                targets    = local.file_match.pods.targets
+                forward_to = [loki.process.pods.receiver]
+              }
+
+              loki.process "pods" {
+                stage.cri {}
+                forward_to = [loki.write.default.receiver]
+              }
+
+              // Host logs: k3s, containerd, kernel and the rest of systemd
+              loki.relabel "journal" {
+                forward_to = []
+
+                rule {
+                  source_labels = ["__journal__systemd_unit"]
+                  target_label  = "unit"
+                }
+                rule {
+                  source_labels = ["__journal__hostname"]
+                  target_label  = "node"
+                }
+                rule {
+                  source_labels = ["__journal_priority_keyword"]
+                  target_label  = "level"
+                }
+              }
+
+              loki.source.journal "node" {
+                path          = "/var/log/journal"
+                max_age       = "12h"
+                labels        = {job = "node/journal"}
+                relabel_rules = loki.relabel.journal.rules
+                forward_to    = [loki.write.default.receiver]
+              }
+
+              loki.write "default" {
+                endpoint {
+                  url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
+                }
+              }
+          mounts:
+            varlog: true
+            extra:
+              - name: machine-id
+                mountPath: /etc/machine-id
+                readOnly: true
+              - name: storage
+                mountPath: /var/lib/alloy
+          # Tail positions survive pod restarts, so logs are not shipped twice.
+          storagePath: /var/lib/alloy
+          resources:
+            limits:
+              cpu: 500m
+              memory: 512Mi
+            requests:
+              cpu: 50m
+              memory: 128Mi
+        configReloader:
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 50Mi
+        controller:
+          volumes:
+            extra:
+              - name: machine-id
+                hostPath:
+                  path: /etc/machine-id
+                  type: File
+              - name: storage
+                hostPath:
+                  path: /var/lib/alloy
+                  type: DirectoryOrCreate
+    repoURL: https://grafana.github.io/helm-charts
+    targetRevision: 1.13.0
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true