diff --git a/prometheus-stack/k8s/alerts.yaml b/prometheus-stack/k8s/alerts.yaml index 489f276..0388386 100644 --- a/prometheus-stack/k8s/alerts.yaml +++ b/prometheus-stack/k8s/alerts.yaml @@ -27,6 +27,33 @@ spec: summary: "Pod is crash looping" description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff." + - alert: ContainerOOMKilled + expr: max_over_time(kube_pod_container_status_terminated_reason{reason="OOMKilled"}[15m]) >= 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Container was OOMKilled" + description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} was killed for exceeding its memory limit. Raise the limit or reduce the workload." + + - alert: ContainerRestartingTooOften + expr: max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total[30m])) > 3 + for: 5m + labels: + severity: warning + annotations: + summary: "Container restarting too often" + description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value }} times in the last 30 minutes." + + - alert: PodEvicted + expr: max_over_time(kube_pod_status_reason{reason="Evicted"}[15m]) >= 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod was evicted" + description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} was evicted, usually for node disk or memory pressure." + - alert: PersistentVolumeClaimFillingUp expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15 for: 15m