feat(add-ons): Add observability stack with kube-prometheus-stack, loki and alloy

2026-09-30 01:26:38 +05:30
parent 148890fb86
commit 576c42554d
18 changed files with 633 additions and 0 deletions
@@ -0,0 +1,117 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "4"
+  name: alloy
+  namespace: argocd
+spec:
+  destination:
+    namespace: monitoring
+    server: https://kubernetes.default.svc
+  project: default
+  source:
+    chart: alloy
+    helm:
+      values: |
+        alloy:
+          configMap:
+            content: |
+              discovery.kubernetes "pods" {
+                role = "pod"
+                selectors {
+                  role  = "pod"
+                  field = "spec.nodeName=" + sys.env("HOSTNAME")
+                }
+              }
+
+              discovery.relabel "pods" {
+                targets = discovery.kubernetes.pods.targets
+
+                rule {
+                  source_labels = ["__meta_kubernetes_namespace"]
+                  target_label  = "namespace"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_name"]
+                  target_label  = "pod"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_container_name"]
+                  target_label  = "container"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"]
+                  regex         = "^;*([^;]+)(;.*)?$"
+                  target_label  = "app"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_node_name"]
+                  target_label  = "node"
+                }
+                rule {
+                  source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+                  separator     = "/"
+                  replacement   = "/var/log/pods/*$1/*.log"
+                  target_label  = "__path__"
+                }
+              }
+
+              local.file_match "pods" {
+                path_targets = discovery.relabel.pods.output
+              }
+
+              loki.source.file "pods" {
+                targets    = local.file_match.pods.targets
+                forward_to = [loki.process.pods.receiver]
+              }
+
+              loki.process "pods" {
+                stage.cri {}
+                // Changes on every rotation and would start a new stream.
+                stage.label_drop {
+                  values = ["filename"]
+                }
+                forward_to = [loki.write.default.receiver]
+              }
+
+              loki.write "default" {
+                endpoint {
+                  url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
+                }
+              }
+          mounts:
+            varlog: true
+            extra:
+              - name: storage
+                mountPath: /var/lib/alloy
+          # Tail positions survive pod restarts, so logs are not shipped twice.
+          storagePath: /var/lib/alloy
+          resources:
+            limits:
+              cpu: 500m
+              memory: 512Mi
+            requests:
+              cpu: 50m
+              memory: 128Mi
+        configReloader:
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+        controller:
+          volumes:
+            extra:
+              - name: storage
+                hostPath:
+                  path: /var/lib/alloy
+                  type: DirectoryOrCreate
+    repoURL: https://grafana.github.io/helm-charts
+    targetRevision: 1.13.0
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true