feat(add-ons): Add observability stack with kube-prometheus-stack, loki and alloy
This commit is contained in:
@@ -0,0 +1,150 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "4"
|
||||
name: alloy
|
||||
namespace: argocd
|
||||
spec:
|
||||
destination:
|
||||
namespace: monitoring
|
||||
server: https://kubernetes.default.svc
|
||||
project: default
|
||||
source:
|
||||
chart: alloy
|
||||
helm:
|
||||
values: |
|
||||
alloy:
|
||||
configMap:
|
||||
content: |
|
||||
// Container logs of pods on this node, tailed from /var/log/pods
|
||||
discovery.kubernetes "pods" {
|
||||
role = "pod"
|
||||
selectors {
|
||||
role = "pod"
|
||||
field = "spec.nodeName=" + sys.env("HOSTNAME")
|
||||
}
|
||||
}
|
||||
|
||||
discovery.relabel "pods" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"]
|
||||
regex = "^;*([^;]+)(;.*)?$"
|
||||
target_label = "app"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
|
||||
separator = "/"
|
||||
replacement = "/var/log/pods/*$1/*.log"
|
||||
target_label = "__path__"
|
||||
}
|
||||
}
|
||||
|
||||
local.file_match "pods" {
|
||||
path_targets = discovery.relabel.pods.output
|
||||
}
|
||||
|
||||
loki.source.file "pods" {
|
||||
targets = local.file_match.pods.targets
|
||||
forward_to = [loki.process.pods.receiver]
|
||||
}
|
||||
|
||||
loki.process "pods" {
|
||||
stage.cri {}
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
// Host logs: k3s, containerd, kernel and the rest of systemd
|
||||
loki.relabel "journal" {
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal__hostname"]
|
||||
target_label = "node"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.journal "node" {
|
||||
path = "/var/log/journal"
|
||||
max_age = "12h"
|
||||
labels = {job = "node/journal"}
|
||||
relabel_rules = loki.relabel.journal.rules
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
mounts:
|
||||
varlog: true
|
||||
extra:
|
||||
- name: machine-id
|
||||
mountPath: /etc/machine-id
|
||||
readOnly: true
|
||||
- name: storage
|
||||
mountPath: /var/lib/alloy
|
||||
# Tail positions survive pod restarts, so logs are not shipped twice.
|
||||
storagePath: /var/lib/alloy
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
configReloader:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 50m
|
||||
memory: 100Mi
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 50Mi
|
||||
controller:
|
||||
volumes:
|
||||
extra:
|
||||
- name: machine-id
|
||||
hostPath:
|
||||
path: /etc/machine-id
|
||||
type: File
|
||||
- name: storage
|
||||
hostPath:
|
||||
path: /var/lib/alloy
|
||||
type: DirectoryOrCreate
|
||||
repoURL: https://grafana.github.io/helm-charts
|
||||
targetRevision: 1.13.0
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
@@ -0,0 +1,195 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "3"
|
||||
name: kube-prometheus-stack
|
||||
namespace: argocd
|
||||
spec:
|
||||
destination:
|
||||
namespace: monitoring
|
||||
server: https://kubernetes.default.svc
|
||||
project: default
|
||||
sources:
|
||||
- chart: kube-prometheus-stack
|
||||
helm:
|
||||
values: |
|
||||
alertmanager:
|
||||
enabled: false
|
||||
# k3s embeds every control-plane component in one process with a shared
|
||||
# metrics registry, so each endpoint also serves the apiserver's series.
|
||||
# Keep those on the apiserver job only.
|
||||
kubelet:
|
||||
serviceMonitor:
|
||||
metricRelabelings:
|
||||
- action: drop
|
||||
sourceLabels: [__name__, le]
|
||||
regex: (csi_operations|storage_operation_duration)_seconds_bucket;(0.25|2.5|15|25|120|600)(\.0)?
|
||||
- action: drop
|
||||
sourceLabels: [__name__]
|
||||
regex: (apiserver|etcd|kine)_.*
|
||||
# Bound to 0.0.0.0 through /etc/rancher/k3s/config.yaml on the node.
|
||||
kubeControllerManager:
|
||||
endpoints:
|
||||
- 192.168.2.5
|
||||
serviceMonitor:
|
||||
metricRelabelings:
|
||||
- action: drop
|
||||
sourceLabels: [__name__]
|
||||
regex: (apiserver|etcd|kine)_.*
|
||||
kubeScheduler:
|
||||
endpoints:
|
||||
- 192.168.2.5
|
||||
serviceMonitor:
|
||||
metricRelabelings:
|
||||
- action: drop
|
||||
sourceLabels: [__name__]
|
||||
regex: (apiserver|etcd|kine)_.*
|
||||
kubeProxy:
|
||||
endpoints:
|
||||
- 192.168.2.5
|
||||
serviceMonitor:
|
||||
metricRelabelings:
|
||||
- action: drop
|
||||
sourceLabels: [__name__]
|
||||
regex: (apiserver|etcd|kine)_.*
|
||||
# k3s runs on SQLite (kine), there is no etcd to scrape.
|
||||
kubeEtcd:
|
||||
enabled: false
|
||||
prometheusOperator:
|
||||
admissionWebhooks:
|
||||
certManager:
|
||||
enabled: true
|
||||
resources:
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
prometheusConfigReloader:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
kube-state-metrics:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 64Mi
|
||||
prometheus-node-exporter:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 32Mi
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
retention: 90d
|
||||
retentionSize: 45GB
|
||||
# Pick up monitors and rules from every namespace, not just this release.
|
||||
podMonitorSelectorNilUsesHelmValues: false
|
||||
probeSelectorNilUsesHelmValues: false
|
||||
ruleSelectorNilUsesHelmValues: false
|
||||
scrapeConfigSelectorNilUsesHelmValues: false
|
||||
serviceMonitorSelectorNilUsesHelmValues: false
|
||||
# Generous CPU ceiling: WAL replay and compaction burst well above steady state.
|
||||
resources:
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 3Gi
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 1Gi
|
||||
storageSpec:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
grafana:
|
||||
admin:
|
||||
existingSecret: grafana-admin
|
||||
additionalDataSources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
url: http://loki.monitoring.svc.cluster.local:3100
|
||||
envValueFrom:
|
||||
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
|
||||
secretKeyRef:
|
||||
name: grafana-oidc
|
||||
key: client-secret
|
||||
GF_DATABASE_PASSWORD:
|
||||
secretKeyRef:
|
||||
name: grafana-db
|
||||
key: password
|
||||
grafana.ini:
|
||||
database:
|
||||
type: postgres
|
||||
host: postgres-rw.postgres.svc.cluster.local:5432
|
||||
name: grafana
|
||||
user: grafana
|
||||
ssl_mode: require
|
||||
server:
|
||||
root_url: https://grafana.whyredfire.dev
|
||||
auth:
|
||||
# Break-glass admin login stays reachable at /login?disableAutoLogin=true
|
||||
oauth_auto_login: true
|
||||
signout_redirect_url: https://auth.whyredfire.dev/application/o/grafana/end-session/
|
||||
auth.generic_oauth:
|
||||
enabled: true
|
||||
name: Authentik
|
||||
client_id: grafana
|
||||
scopes: openid profile email
|
||||
auth_url: https://auth.whyredfire.dev/application/o/authorize/
|
||||
token_url: https://auth.whyredfire.dev/application/o/token/
|
||||
api_url: https://auth.whyredfire.dev/application/o/userinfo/
|
||||
use_pkce: true
|
||||
login_attribute_path: preferred_username
|
||||
role_attribute_path: contains(groups, 'admins') && 'Admin'
|
||||
role_attribute_strict: true
|
||||
# State lives in Postgres; the pod itself is disposable.
|
||||
persistence:
|
||||
enabled: false
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 256Mi
|
||||
sidecar:
|
||||
dashboards:
|
||||
searchNamespace: ALL
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
testFramework:
|
||||
enabled: false
|
||||
repoURL: https://prometheus-community.github.io/helm-charts
|
||||
targetRevision: 91.8.2
|
||||
- path: add-ons/kube-prometheus-stack
|
||||
repoURL: https://gitea.whyredfire.dev/whyredfire/gitops.git
|
||||
targetRevision: main
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
@@ -1,10 +1,13 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- alloy.yaml
|
||||
- argocd-dashboard.yaml
|
||||
- authentik.yaml
|
||||
- cert-manager.yaml
|
||||
- cnpg.yaml
|
||||
- kube-prometheus-stack.yaml
|
||||
- loki.yaml
|
||||
- postgres.yaml
|
||||
- sealed-secrets.yaml
|
||||
- traefik.yaml
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "4"
|
||||
name: loki
|
||||
namespace: argocd
|
||||
spec:
|
||||
destination:
|
||||
namespace: monitoring
|
||||
server: https://kubernetes.default.svc
|
||||
project: default
|
||||
sources:
|
||||
- chart: loki
|
||||
helm:
|
||||
values: |
|
||||
deploymentMode: Monolithic
|
||||
loki:
|
||||
auth_enabled: false
|
||||
commonConfig:
|
||||
replication_factor: 1
|
||||
schemaConfig:
|
||||
configs:
|
||||
- from: "2026-09-30"
|
||||
store: tsdb
|
||||
object_store: s3
|
||||
schema: v13
|
||||
index:
|
||||
prefix: loki_index_
|
||||
period: 24h
|
||||
storage:
|
||||
type: s3
|
||||
bucketNames:
|
||||
chunks: loki
|
||||
ruler: loki
|
||||
s3:
|
||||
endpoint: http://garage.garage.svc.cluster.local:3900
|
||||
region: garage
|
||||
accessKeyId: ${LOKI_S3_ACCESS_KEY_ID}
|
||||
secretAccessKey: ${LOKI_S3_SECRET_ACCESS_KEY}
|
||||
s3ForcePathStyle: true
|
||||
insecure: true
|
||||
ingester:
|
||||
chunk_encoding: zstd
|
||||
compactor:
|
||||
retention_enabled: true
|
||||
delete_request_store: s3
|
||||
limits_config:
|
||||
retention_period: 2160h
|
||||
singleBinary:
|
||||
replicas: 1
|
||||
extraEnvFrom:
|
||||
- secretRef:
|
||||
name: loki-s3
|
||||
persistence:
|
||||
size: 10Gi
|
||||
resources:
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 2Gi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
# Memory limits sit ~20% above memcached's allocation for connection overhead.
|
||||
chunksCache:
|
||||
allocatedMemory: 512
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 614Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 614Mi
|
||||
resultsCache:
|
||||
allocatedMemory: 128
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 154Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 154Mi
|
||||
memcachedExporter:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
# Loki's rules sidecar
|
||||
sidecar:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
# Grafana and Alloy talk to Loki directly, there is a single tenant.
|
||||
gateway:
|
||||
enabled: false
|
||||
lokiCanary:
|
||||
enabled: false
|
||||
test:
|
||||
enabled: false
|
||||
backend:
|
||||
replicas: 0
|
||||
read:
|
||||
replicas: 0
|
||||
write:
|
||||
replicas: 0
|
||||
repoURL: https://grafana-community.github.io/helm-charts
|
||||
targetRevision: 18.13.7
|
||||
- path: add-ons/loki
|
||||
repoURL: https://gitea.whyredfire.dev/whyredfire/gitops.git
|
||||
targetRevision: main
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
Reference in New Issue
Block a user