feat(monitoring): align storage to live pvc, add loki datasource and alerts

Match grafana/alertmanager storageClass to live PVCs to avoid immutable-field failures. Add Loki datasource and LokiDown/AlloyDown alerts.

prometheus-stack/k8s/alerts.yaml

@@ -66,3 +66,21 @@ spec:
           annotations:
             summary: "Node memory pressure"
             description: "Node {{ $labels.instance }} has used more than 90% of memory for 15 minutes."
+
+        - alert: LokiDown
+          expr: kube_statefulset_status_replicas_unavailable{statefulset="loki"} > 0 or kube_deployment_status_replicas_unavailable{deployment="loki-gateway"} > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Loki is down"
+            description: "Loki in namespace {{ $labels.namespace }} has unavailable replicas for more than 10 minutes. Logs are not queryable."
+
+        - alert: AlloyDown
+          expr: kube_daemonset_status_number_unavailable{daemonset="alloy"} > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Alloy is down"
+            description: "Alloy DaemonSet in namespace {{ $labels.namespace }} has {{ $value }} unavailable pods for more than 10 minutes. Pod logs are not being shipped to Loki."

prometheus-stack/k8s/grafana-values.yaml

@@ -14,8 +14,11 @@ grafana:
 
   persistence:
     enabled: true
-    storageClassName: local-path-retain
+    # Matches live PVC (10Gi/local-path). Retain migration is a separate task
-    size: 20Gi
+    # with data migration (see storage-audit doc); do NOT change SC/size here
+    # without migrating, helm upgrade fails on immutable PVC fields.
+    storageClassName: local-path
+    size: 10Gi
 
   ingress:
     enabled: false
@@ -23,6 +26,12 @@ grafana:
   service:
     port: 80
 
+  additionalDataSources:
+    - name: Loki
+      type: loki
+      url: http://loki-gateway.prometheus.svc.cluster.local
+      access: proxy
+
 prometheus:
   prometheusSpec:
     retention: 60d
@@ -30,7 +39,8 @@ prometheus:
     storageSpec:
       volumeClaimTemplate:
         spec:
-          storageClassName: "local-path-retain"
+          # Matches live PVC, see note on grafana.persistence above.
+          storageClassName: "local-path"
           accessModes:
             - ReadWriteOnce
           resources:
@@ -48,12 +58,13 @@ alertmanager:
     storage:
       volumeClaimTemplate:
         spec:
-          storageClassName: local-path-retain
+          # Matches live PVC (20Gi), see note on grafana.persistence above.
+          storageClassName: local-path
           accessModes:
             - ReadWriteOnce
           resources:
             requests:
-              storage: 10Gi
+              storage: 20Gi
 
 defaultRules:
   disabled:

prometheus-stack/k8s/ingress.yaml

@@ -12,7 +12,6 @@ spec:
       middlewares:
         - name: crowdsec-bouncer
           namespace: crowdsec
-        - name: "security-chain@file"
       services:
         - name: prometheus-stack-grafana
           port: 80