feat(alerts): cover OOM kills, restart loops and evictions
The OOMKilled container behind the immich crash loop was invisible: PodCrashLooping only fires once kubelet has already given up and started the backoff.
This commit is contained in:
1 parent
f5b2f89f38
commit
74adf38d63
1 file changed
+27
@@ -27,6 +27,33 @@ spec:
|
||||
summary: "Pod is crash looping"
|
||||
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff."
|
||||
|
||||
- alert: ContainerOOMKilled
|
||||
expr: max_over_time(kube_pod_container_status_terminated_reason{reason="OOMKilled"}[15m]) >= 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Container was OOMKilled"
|
||||
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} was killed for exceeding its memory limit. Raise the limit or reduce the workload."
|
||||
|
||||
- alert: ContainerRestartingTooOften
|
||||
expr: max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total[30m])) > 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Container restarting too often"
|
||||
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value }} times in the last 30 minutes."
|
||||
|
||||
- alert: PodEvicted
|
||||
expr: max_over_time(kube_pod_status_reason{reason="Evicted"}[15m]) >= 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod was evicted"
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} was evicted, usually for node disk or memory pressure."
|
||||
|
||||
- alert: PersistentVolumeClaimFillingUp
|
||||
expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15
|
||||
for: 15m
|
||||
|
||||
Reference in new issue
Block a user