feat(alerts): cover OOM kills, restart loops and evictions
The OOMKilled container behind the immich crash loop was invisible: PodCrashLooping only fires once kubelet has already given up and started the backoff.
This commit is contained in:
1 parent
f5b2f89f38
commit
74adf38d63
1 file changed
+27
@@ -27,6 +27,33 @@ spec:
|
|||||||
summary: "Pod is crash looping"
|
summary: "Pod is crash looping"
|
||||||
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff."
|
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff."
|
||||||
|
|
||||||
|
- alert: ContainerOOMKilled
|
||||||
|
expr: max_over_time(kube_pod_container_status_terminated_reason{reason="OOMKilled"}[15m]) >= 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Container was OOMKilled"
|
||||||
|
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} was killed for exceeding its memory limit. Raise the limit or reduce the workload."
|
||||||
|
|
||||||
|
- alert: ContainerRestartingTooOften
|
||||||
|
expr: max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total[30m])) > 3
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Container restarting too often"
|
||||||
|
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value }} times in the last 30 minutes."
|
||||||
|
|
||||||
|
- alert: PodEvicted
|
||||||
|
expr: max_over_time(kube_pod_status_reason{reason="Evicted"}[15m]) >= 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Pod was evicted"
|
||||||
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} was evicted, usually for node disk or memory pressure."
|
||||||
|
|
||||||
- alert: PersistentVolumeClaimFillingUp
|
- alert: PersistentVolumeClaimFillingUp
|
||||||
expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15
|
expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15
|
||||||
for: 15m
|
for: 15m
|
||||||
|
|||||||
Reference in new issue
Block a user