feat(crowdsec): switch agent to loki acquisition with static identity

Read traefik logs from Loki instead of file tail. Static machine identity via pre-created secret stops 403 register races. Add janitor cronjob, dashboards, whitelists and metrics.

crowdsec/k8s/crowdsec-values.yaml

@@ -1,57 +1,102 @@
 container_runtime: containerd
+
 agent:
+  acquisition: []
+  additionalAcquisition:
+    - labels:
+        type: traefik
+      limit: 1000
+      query: |
+        {namespace="traefik"}
+      source: loki
+      url: http://loki.prometheus.svc.cluster.local:3100/
+      wait_for_ready: 30s
   env:
     - name: COLLECTIONS
-      value: "crowdsecurity/traefik crowdsecurity/base-http-scenarios"
+      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
     - name: DISABLE_COLLECTIONS
-      value: "crowdsecurity/linux crowdsecurity/sshd"
+      value: crowdsecurity/sshd
-
+  metrics:
-  acquisition:
+    enabled: true
-    - namespace: traefik
+    serviceMonitor:
-      podName: "*traefik*"
+      additionalLabels:
-      program: traefik
+        release: prometheus-stack
-      poll_without_inotify: true
+      enabled: true
-
+  # Static machine identity: agent pods mount pre-created LAPI credentials
+  # (Secret crowdsec-agent-credentials, key local_api_credentials.yaml)
+  # at the exact path the agent entrypoint expects. Together with the
+  # patched register-init (enforced by janitor-cronjob.yaml) the agent
+  # never calls `cscli lapi register` in steady state, so pod names,
+  # restarts and reboots can no longer break it.
+  extraVolumes:
+    - name: static-creds
+      secret:
+        secretName: crowdsec-agent-credentials
+        items:
+          - key: local_api_credentials.yaml
+            path: local_api_credentials.yaml
+  extraVolumeMounts:
+    - name: static-creds
+      mountPath: /tmp_config/local_api_credentials.yaml
+      subPath: local_api_credentials.yaml
+      readOnly: true
   resources:
-    requests:
-      cpu: 50m
-      memory: 100Mi
     limits:
       cpu: 200m
       memory: 500Mi
+    requests:
+      cpu: 50m
+      memory: 100Mi
+
+config:
+  parsers:
+    s02-enrich:
+      mobile-whitelist.yaml: |
+        name: forust/mobile-whitelist
+        description: "Whitelist SWAN/4ka mobile network"
+        whitelist:
+          reason: "Mobile IP whitelist"
+          cidr:
+            - "84.245.64.0/18"
+
+  postoverflows:
+    s01-whitelist:
+      home-dynamic-ip.yaml: |
+        name: forust/home-dynamic-ip
+        description: "Whitelist home dynamic IP"
+        whitelist:
+          reason: "Home dynamic IP"
+          expression:
+            - evt.Overflow.Alert.Source.IP in LookupHost("ddns.forust.xyz")
 
 lapi:
   env:
     - name: COLLECTIONS
-      value: "crowdsecurity/traefik crowdsecurity/base-http-scenarios"
+      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
     - name: DISABLE_COLLECTIONS
-      value: "crowdsecurity/linux crowdsecurity/sshd"
+      value: crowdsecurity/linux crowdsecurity/sshd
-  service:
+  metrics:
-    type: ClusterIP
+    enabled: true
-  persistentVolume:
+    serviceMonitor:
-    data:
+      additionalLabels:
+        release: prometheus-stack
       enabled: true
-      storageClassName: local-path-retain
+  persistentVolume:
-      size: 1Gi
     config:
       enabled: true
-      storageClassName: local-path-retain
       size: 100Mi
-  storeLAPICscliCredentialsInSecret: true
+      storageClassName: local-path-retain
+    data:
+      enabled: true
+      size: 1Gi
+      storageClassName: local-path-retain
   resources:
-    requests:
-      cpu: 50m
-      memory: 150Mi
     limits:
       cpu: 400m
       memory: 500Mi
-
+    requests:
-metrics:
+      cpu: 50m
-  enabled: true
+      memory: 150Mi
-  serviceMonitor:
+  service:
-    additionalLabels:
+    type: ClusterIP
-      release: prometheus-stack
+  storeLAPICscliCredentialsInSecret: true
-    enabled: true
-    interval: 30s
-    scrapeTimeout: 10s
-    namespace: prometheus

crowdsec/k8s/grafana-dashboards.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+data:
+  crowdsec-overview.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph (old)\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"timeseries\",\n      \"name\": \"Time series\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 24,\n      \"panels\": [],\n      \"title\": \"Summary\",\n      \"type\": \"row\"\n    },\n    {\n      \"cacheTimeout\": null,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [\n            {\n              \"options\": {\n                \"match\": \"null\",\n                \"result\": {\n                  \"text\": \"N/A\"\n                }\n              },\n              \"type\": \"special\"\n            }\n          ],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"#E02F44\",\n                \"value\": null\n              },\n              {\n                \"color\": \"#E02F44\",\n                \"value\": 10\n              },\n              {\n                \"color\": \"#299c46\",\n                \"value\": 10\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 6,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"interval\": null,\n      \"links\": [],\n      \"maxDataPoints\": 100,\n      \"options\": {\n        \"colorMode\": \"background\",\n        \"graphMode\": \"none\",\n        \"justifyMode\": \"auto\",\n        \"orientation\": \"horizontal\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"text\": {},\n        \"textMode\": \"auto\"\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"count(cs_info)\",\n          \"interval\": \"\",\n          \"legendFormat\": \"\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"timeFrom\": null,\n      \"timeShift\": null,\n      \"title\": \"Running Crowdsec\",\n      \"transparent\": true,\n      \"type\": \"stat\"\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"decimals\": 1,\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"links\": []\n        },\n        \"overrides\": []\n      },\n      \"fill\": 1,\n      \"fillGradient\": 0,\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 18,\n        \"x\": 6,\n        \"y\": 1\n      },\n      \"hiddenSeries\": false,\n      \"id\": 8,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": false,\n        \"max\": false,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": \"total\",\n        \"sortDesc\": true,\n        \"total\": true,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"nullPointMode\": \"null\",\n      \"options\": {\n        \"alertThreshold\": true\n      },\n      \"percentage\": false,\n      \"pluginVersion\": \"8.1.2\",\n      \"pointradius\": 2,\n      \"points\": false,\n      \"renLine truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: '1'
+  name: crowdsec-crowdsec-overview
+  namespace: prometheus
+---
+apiVersion: v1
+data:
+  crowdsec-lapi-metrics.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915193937,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 10,\n      \"panels\": [],\n      \"title\": \"Agents\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              },\n              {\n                \"color\": \"red\",\n                \"value\": 80\n              }\n            ]\n          }\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"vertical\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"repeat\": \"query0\",\n      \"repeatDirection\": \"h\",\n      \"targets\": [\n        {\n          \"exemplar\": false,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\", instance=\\\"$lapi\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Agents Login\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 1\n      },\n      \"id\": 6,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"auto\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Heartbeat\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 12,\n      \"panels\": [],\n      \"title\": \"Decisions\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n  Line truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: '1'
+  name: crowdsec-crowdsec-lapi-metrics
+  namespace: prometheus
+---
+apiVersion: v1
+data:
+  crowdsec-insight.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"gauge\",\n      \"name\": \"Gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915159751,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 22,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"color\": {\n                \"mode\": \"thresholds\"\n              },\n              \"mappings\": [\n                {\n                  \"options\": {\n                    \"match\": \"null\",\n                    \"result\": {\n                      \"text\": \"N/A\"\n                    }\n                  },\n                  \"type\": \"special\"\n                }\n              ],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  },\n                  {\n                    \"color\": \"red\",\n                    \"value\": 80\n                  }\n                ]\n              },\n              \"unit\": \"dateTimeAsIso\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 2,\n            \"y\": 1\n          },\n          \"id\": 2,\n          \"interval\": null,\n          \"links\": [],\n          \"maxDataPoints\": 100,\n          \"options\": {\n            \"colorMode\": \"none\",\n            \"graphMode\": \"none\",\n            \"justifyMode\": \"auto\",\n            \"orientation\": \"horizontal\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"lastNotNull\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\n            \"text\": {},\n            \"textMode\": \"auto\"\n          },\n          \"pluginVersion\": \"8.1.2\",\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"(process_start_time_seconds{instance=\\\"$instance\\\"})*1000\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Up since\",\n          \"type\": \"stat\"\n        },\n        {\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"displayName\": \"\",\n              \"mappings\": [],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  }\n                ]\n              },\n              \"unit\": \"decbytes\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 7,\n            \"y\": 1\n          },\n          \"id\": 4,\n          \"options\": {\n            \"orientation\": \"auto\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"mean\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\Line truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: '1'
+  name: crowdsec-crowdsec-insight
+  namespace: prometheus

crowdsec/k8s/janitor-cronjob.yaml

@@ -0,0 +1,195 @@
+# CrowdSec self-healing: static machine identity + enforcement loops.
+#
+# Problem it fixes: the chart's agent init container runs
+#   `cscli lapi register --machine "$POD_NAME" ...`
+# unconditionally. Credentials live in an emptyDir, the machine row lives
+# in LAPI's persistent DB. Any init re-run for an already-known pod name
+# (kubelet restart, node reboot) dies with
+#   403 Forbidden: user '<pod>' already exist
+# and the DaemonSet pod sticks in Init forever. Every DS restart also
+# leaves an orphan machine row that is never cleaned.
+#
+# Design (name-independent):
+#   * Agent identity is a STATIC machine `crowdsec-agent-workstation`
+#     whose password lives in Secret `crowdsec-agent-credentials`
+#     (created once, manually - like all other secrets in this repo).
+#     The secret is mounted into agent pods at
+#     /tmp_config/local_api_credentials.yaml (see extraVolumeMounts in
+#     crowdsec-values.yaml), which is exactly the path the agent's main
+#     container copies into place at startup.
+#   * The DS init command is patched (strategic merge, by container name)
+#     to SKIP registration when that file exists, keeping the legacy
+#     register path only as fallback. Detection marker in the patched
+#     command: `[ -s /tmp_config`.
+#   * This CronJob enforces the desired state hourly, so recovery is
+#     automatic even after `helm upgrade` reverts the DS patch or the
+#     LAPI database is wiped:
+#       1. patch DS init if it still has the unconditional register
+#          (no-op otherwise - no restart churn);
+#       2. prune machines with no heartbeat for 2h (orphan hygiene);
+#       3. ensure the static machine exists, recreating it with the
+#          Secret password if missing (agent retry loops reconnect
+#          on their own - same name + same password);
+#       4. prune bouncer entries idle for 30d.
+#
+# Manual apply (crowdsec/k8s is NOT managed by deploy.yaml):
+#   kubectl apply -f crowdsec/k8s/janitor-cronjob.yaml
+# Force a run:
+#   kubectl create job -n crowdsec --from=cronjob/crowdsec-janitor janitor-now
+#
+# Helm upgrades: the janitor's strategic patch puts the DS field under
+# the `kubectl-patch` field manager, so a plain `helm upgrade` FAILS
+# with an SSA conflict on initContainers[].command. Procedure:
+#   1. revert init to chart state (kills the conflict):
+#        helm template crowdsec crowdsec/crowdsec --version <ver> \
+#          -n crowdsec -f crowdsec/k8s/crowdsec-values.yaml > /tmp/r.yaml
+#        python3 -c "import yaml,json; ..." # build revert patch from
+#        the rendered DaemonSet init command, then
+#        kubectl patch ds crowdsec-agent -n crowdsec \
+#          --type strategic -p "\$(cat /tmp/revert_patch.json)"
+#   2. helm upgrade --install crowdsec ... (no --force needed)
+#   3. janitor-now right away (upgrade reverts init; new pods would
+#      sit in Init until the next hourly run otherwise).
+#
+# One-time bootstrap (order matters):
+#   1. Create Secret + static machine (see commands in chat).
+#   2. Apply this file, trigger janitor-now, wait for agent 1/1.
+#   3. One-time orphan cleanup:
+#        kubectl exec -n crowdsec deploy/crowdsec-lapi -- \
+#          cscli machines prune --duration 1h --force
+#   4. Only then `helm upgrade` crowdsec with the extraVolumes values.
+#      Upgrade reverts the DS patch; trigger janitor-now right after it
+#      (otherwise new pods sit in Init until the next hourly run, then
+#      self-heal anyway).
+#
+# Password rotation: update the Secret, delete the machine
+# (`cscli machines delete crowdsec-agent-workstation`), trigger
+# janitor-now (recreates it), then `kubectl rollout restart
+# ds/crowdsec-agent -n crowdsec` (agent reads the file at startup only).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: ["apps"]
+    resources: ["daemonsets"]
+    verbs: ["get", "patch"]
+  # `kubectl exec deploy/<name>` resolves deploy -> replicaset -> pod,
+  # which needs read access to these (exec itself is pods/exec above).
+  - apiGroups: ["apps"]
+    resources: ["deployments", "replicasets"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+subjects:
+  - kind: ServiceAccount
+    name: crowdsec-janitor
+    namespace: crowdsec
+roleRef:
+  kind: Role
+  name: crowdsec-janitor
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+spec:
+  schedule: "17 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 300
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/part-of: crowdsec
+        spec:
+          serviceAccountName: crowdsec-janitor
+          restartPolicy: OnFailure
+          containers:
+            - name: janitor
+              # Same image the chart itself uses for registration jobs;
+              # IfNotPresent so it works while the node is offline
+              # (layer cached from the chart install).
+              image: alpine/kubectl:latest
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: AGENT_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: crowdsec-agent-credentials
+                      key: password
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  LAPI_EXEC="kubectl exec -n crowdsec deploy/crowdsec-lapi --"
+                  echo "== 1. enforce patched agent init =="
+                  CUR=$(kubectl get ds crowdsec-agent -n crowdsec \
+                    -o jsonpath='{.spec.template.spec.initContainers[0].command[2]}')
+                  case "$CUR" in
+                    *'-s /tmp_config'*)
+                      echo "init already patched"
+                      ;;
+                    *)
+                      echo "patching init"
+                      WAIT='until nc "$LAPI_HOST" "$LAPI_PORT" -z'
+                      WAIT="$WAIT; do echo waiting for lapi to start; sleep 5; done"
+                      LINK='ln -s /staging/etc/crowdsec /etc/crowdsec'
+                      REG='cscli lapi register --machine "$USERNAME"'
+                      REG="$REG -u \"\$LAPI_URL\" --token \"\$REGISTRATION_TOKEN\""
+                      CREDS=/tmp_config/local_api_credentials.yaml
+                      CMD="$WAIT; $LINK; [ -s $CREDS ] || {"
+                      CMD="$CMD $REG && cp"
+                      CMD="$CMD /etc/crowdsec/local_api_credentials.yaml $CREDS; }"
+                      ESC=$(printf '%s' "$CMD" | sed 's/"/\\"/g')
+                      PATCH='{"spec":{"template":{"spec":{"initContainers":'
+                      PATCH=$PATCH'[{"name":"wait-for-lapi-and-register",'
+                      PATCH=$PATCH'"command":["sh","-c","'$ESC'"]}]}}}}'
+                      kubectl patch ds crowdsec-agent -n crowdsec \
+                        --type strategic -p "$PATCH"
+                      ;;
+                  esac
+                  echo "== 2. prune orphan machines (no heartbeat for 2h) =="
+                  $LAPI_EXEC cscli machines prune --duration 2h --force
+                  echo "== 3. ensure static machine exists =="
+                  if $LAPI_EXEC cscli machines inspect \
+                    crowdsec-agent-workstation >/dev/null 2>&1; then
+                    echo "static machine present"
+                  else
+                    echo "recreating static machine"
+                    $LAPI_EXEC cscli machines add crowdsec-agent-workstation \
+                      --password "$AGENT_PASSWORD" --force
+                  fi
+                  echo "== 4. prune stale bouncers (no pull for 30d) =="
+                  $LAPI_EXEC cscli bouncers prune -d 720h --force

crowdsec/k8s/network-policy.yaml

@@ -25,3 +25,10 @@ spec:
       ports:
         - protocol: TCP
           port: 8080
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: prometheus
+      ports:
+        - protocol: TCP
+          port: 6060