diff --git a/.gitea/workflows/deploy-lib.sh b/.gitea/workflows/deploy-lib.sh index e89b349..fa7452e 100644 --- a/.gitea/workflows/deploy-lib.sh +++ b/.gitea/workflows/deploy-lib.sh @@ -994,8 +994,7 @@ traefik_routed_hosts() { # # Except that a 404 is not evidence on its own. A router Traefik refused to # build answers with the same 404 and nothing behind it, so a middleware that -# fails to load -- the crowdsec bouncer, which Traefik disables silently when -# it cannot fetch the plugin -- takes down every route that referenced it while +# fails to load takes down every route that referenced it while # this stage reports `ok` for all of them. No status code separates those two # cases, so ask Traefik which routes it built and fail on the difference. stage_smoke() { diff --git a/adguardhome/k8s/ingress.yaml b/adguardhome/k8s/ingress.yaml index 2832a7f..e3a736c 100644 --- a/adguardhome/k8s/ingress.yaml +++ b/adguardhome/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`dns.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: adguard-service port: 3000 diff --git a/authentik/k8s/ingress.yaml b/authentik/k8s/ingress.yaml index 2294f35..e729b70 100644 --- a/authentik/k8s/ingress.yaml +++ b/authentik/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`auth.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: authentik-server-service port: 9000 diff --git a/checkmk/k8s/ingress.yaml b/checkmk/k8s/ingress.yaml index c736bc7..b3f0dbb 100644 --- a/checkmk/k8s/ingress.yaml +++ b/checkmk/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`cmk.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: checkmk-service port: 5000 diff --git a/crowdsec/k8s/crowdsec-middleware.yaml b/crowdsec/k8s/crowdsec-middleware.yaml deleted file mode 100644 index 6d81c98..0000000 --- a/crowdsec/k8s/crowdsec-middleware.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# crowdsec/k8s is NOT managed by deploy.yaml - apply this by hand, and apply it -# together with a restart: -# kubectl apply -f crowdsec/k8s/crowdsec-middleware.yaml -# kubectl -n traefik rollout restart deploy/traefik -# -# The restart is not optional. In stream mode the plugin runs a package-level -# ticker goroutine (handleStreamTicker over the isCrowdsecStreamHealthy and -# updateFailure globals) that no reconfiguration stops. Applying a change -# wedges the instance: every route referencing it answers 404 and traefik logs -# 'invalid middleware crowdsec-crowdsec-bouncer@kubernetescrd' until the pod is -# replaced. Re-applying the previous config does NOT recover it, and the config -# is not the cause - a valid CIDR cannot fail NewChecker, which is a plain -# net.ParseCIDR. Only a new pod clears it. Measured cost: ~35s down for all -# 20 hosts behind this middleware. -apiVersion: traefik.io/v1alpha1 -kind: Middleware -metadata: - name: crowdsec-bouncer - namespace: crowdsec -spec: - plugin: - crowdsec-bouncer: - enabled: true - LogLevel: INFO - # `live` blocked on a `GET /v1/decisions` per request, so a burst - # saturated the LAPI and the plugin 403'd IPs that were never banned. - # v1.3.3 ignores UpdateMaxFailure in `live`, so fail-open is only - # reachable in stream mode, which polls into a cache instead - no - # per-request call to saturate. 15s rather than the 60s default: the - # deploy runner shares one public IP with the house, so this bounds - # both how late a ban lands and how long a lifted one lingers. - CrowdsecMode: stream - UpdateIntervalSeconds: 15 - # -1 = never block because the LAPI is unreachable. In v1.3.3 - # handleStreamTicker only clears isCrowdsecStreamHealthy when - # updateMaxFailure != -1, and ServeHTTP 403s once it is false, so this - # makes a CrowdSec outage mean "no protection", not "every site 403". - UpdateMaxFailure: -1 - CrowdsecLapiScheme: http - CrowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080 - CrowdsecLapiKeyFile: "/etc/traefik/secrets/traefik-api-key" - # Bypasses the bouncer and the decision cache, no LAPI round-trip. - # Keep in sync with forust/local-network in crowdsec-values.yaml. - ClientTrustedIPs: - - "127.0.0.0/8" - - "10.0.0.0/8" - - "172.16.0.0/12" - - "192.168.0.0/16" - - "100.64.0.0/10" - - "169.254.0.0/16" - - "fc00::/7" - - "fe80::/10" - # The mobile operator range from forust/mobile-whitelist, repeated - # deliberately rather than relying on the parser whitelist alone. - # That whitelist drops the event before it reaches a bucket, so no - # decision is ever created - but it is one config away from not - # firing, and the bouncer would then enforce a ban that was never - # justified. This is the last line: even a decision that exists for - # any reason is not served against the phone. - - "84.245.64.0/18" - # The name is HTTPTimeoutSeconds, an int in seconds (min 1) - there is - # no CrowdsecLapiTimeout, and an unrecognised key is silently dropped, - # which is how this sat at the 10s default. Nothing rides on it per - # request any more, so this only bounds the stream pull - and too low - # is the dangerous direction: the LAPI needs ~2s to answer - # /v1/decisions/stream, and a pull that times out leaves the ban cache - # frozen at its startup contents ("failed sending new decisions"), - # i.e. new bans silently never apply. Keep it above the pull latency. - HTTPTimeoutSeconds: 10 diff --git a/crowdsec/k8s/crowdsec-values.yaml b/crowdsec/k8s/crowdsec-values.yaml index ed51279..98d0bf3 100644 --- a/crowdsec/k8s/crowdsec-values.yaml +++ b/crowdsec/k8s/crowdsec-values.yaml @@ -16,6 +16,12 @@ agent: value: crowdsecurity/traefik crowdsecurity/base-http-scenarios - name: DISABLE_COLLECTIONS value: crowdsecurity/sshd + # Bans on 401/403 bursts hurt more than they protect: with L3 enforcement + # a false positive cuts the IP off everything (SSH included), and past + # incidents show legit automation (deploy runner, mesh peers, registry + # pulls) tripping this probe. Probing/XSS/SQLi/CVE scenarios stay. + - name: DISABLE_SCENARIOS + value: crowdsecurity/http-generic-bf metrics: enabled: true serviceMonitor: diff --git a/dockmon/k8s/ingress.yaml b/dockmon/k8s/ingress.yaml index 50d2fbf..0fa79bf 100644 --- a/dockmon/k8s/ingress.yaml +++ b/dockmon/k8s/ingress.yaml @@ -18,8 +18,6 @@ spec: - match: Host(`dockmon.forust.xyz`) kind: Rule middlewares: - - name: crowdsec-bouncer - namespace: crowdsec - name: security-headers@file services: - name: dockmon-service diff --git a/downtify/k8s/ingress.yaml b/downtify/k8s/ingress.yaml index e6711a5..00b5087 100644 --- a/downtify/k8s/ingress.yaml +++ b/downtify/k8s/ingress.yaml @@ -10,8 +10,6 @@ spec: - match: Host(`downtify.forust.xyz`) kind: Rule middlewares: - - name: crowdsec-bouncer - namespace: crowdsec - name: security-chain@file services: - name: downtify-service diff --git a/gitea/k8s/ingress.yaml b/gitea/k8s/ingress.yaml index 3b7bec8..835d0b6 100644 --- a/gitea/k8s/ingress.yaml +++ b/gitea/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`gitea.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: gitea-service port: 3000 diff --git a/headscale/k8s/routing/ingress.yaml b/headscale/k8s/routing/ingress.yaml index cbdc496..b1e86cc 100644 --- a/headscale/k8s/routing/ingress.yaml +++ b/headscale/k8s/routing/ingress.yaml @@ -23,17 +23,11 @@ spec: port: 8080 - match: Host(`hs.forust.xyz`) && PathPrefix(`/admin`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: headscale-ui-external port: 80 - match: Host(`hs.forust.xyz`) && PathPrefix(`/metrics`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: headscale-server-external port: 9090 @@ -53,8 +47,6 @@ spec: kind: Rule middlewares: - name: headplane-prefix - - name: crowdsec-bouncer - namespace: crowdsec services: - name: headplane-external port: 3000 diff --git a/homepages/k8s/gateway.yaml b/homepages/k8s/gateway.yaml new file mode 100644 index 0000000..4b21824 --- /dev/null +++ b/homepages/k8s/gateway.yaml @@ -0,0 +1,31 @@ +# Gateway API PoC for homepages. Lives next to the TLS secrets so +# certificateRefs stay same-namespace and no ReferenceGrant is needed. +# Listener ports must match the Traefik entryPoints (80/443), +# otherwise Traefik marks the listener Invalid. +# Local .internal hosts are deliberately left on IngressRoute, +# only prod is migrated here. +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: homepages + namespace: homepages +spec: + gatewayClassName: traefik + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same + - name: https + protocol: HTTPS + port: 443 + tls: + mode: Terminate + certificateRefs: + - name: forust-homepage-prod-tls + - name: xdfnx-homepage-prod-tls + allowedRoutes: + namespaces: + from: Same diff --git a/homepages/k8s/httproute.yaml b/homepages/k8s/httproute.yaml new file mode 100644 index 0000000..7b6c3d7 --- /dev/null +++ b/homepages/k8s/httproute.yaml @@ -0,0 +1,69 @@ +# PoC: homepages prod hosts via Gateway API. +# Runs alongside k8s/ingress.yaml - delete the prod IngressRoutes only after verification. +# There are no local .internal hosts here, they stay on the local IngressRoute. +# No per-route security middlewares: L3 enforcement moved to the host +# firewall bouncer, so HTTPRoutes stay clean. +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: homepages-http-redirect + namespace: homepages +spec: + parentRefs: + - name: homepages + kind: Gateway + sectionName: http + hostnames: + - forust.xyz + - www.forust.xyz + - xdfnx.cfd + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: forust-homepage-https + namespace: homepages +spec: + parentRefs: + - name: homepages + kind: Gateway + sectionName: https + hostnames: + - forust.xyz + - www.forust.xyz + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: forust-homepage-service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: xdfnx-homepage-https + namespace: homepages +spec: + parentRefs: + - name: homepages + kind: Gateway + sectionName: https + hostnames: + - xdfnx.cfd + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: xdfnx-homepage-service + port: 80 diff --git a/homepages/k8s/ingress.yaml b/homepages/k8s/ingress.yaml index 049d546..92a5633 100644 --- a/homepages/k8s/ingress.yaml +++ b/homepages/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`forust.xyz`) || Host(`www.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec priority: 10 services: - name: forust-homepage-service @@ -49,9 +46,6 @@ spec: routes: - match: Host(`xdfnx.cfd`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: xdfnx-homepage-service port: 80 diff --git a/immich/k8s/ingress.yaml b/immich/k8s/ingress.yaml index 322f959..97ce776 100644 --- a/immich/k8s/ingress.yaml +++ b/immich/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`immich.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: immich-service port: 2283 diff --git a/kener/k8s/ingress.yaml b/kener/k8s/ingress.yaml index 13d2403..dfa186f 100644 --- a/kener/k8s/ingress.yaml +++ b/kener/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`status.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: kener-service port: 3000 diff --git a/n8n/k8s/ingress.yaml b/n8n/k8s/ingress.yaml index b8d7078..a7bbdf6 100644 --- a/n8n/k8s/ingress.yaml +++ b/n8n/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`n8n.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: n8n-service port: 5678 diff --git a/netbird/k8s/ingress.yaml b/netbird/k8s/ingress.yaml index 663148c..7aaae6f 100644 --- a/netbird/k8s/ingress.yaml +++ b/netbird/k8s/ingress.yaml @@ -32,9 +32,6 @@ spec: - match: Host(`nb.forust.xyz`) kind: Rule priority: 1 - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: netbird-dashboard-service port: 80 diff --git a/netbox/k8s/ingress.yaml b/netbox/k8s/ingress.yaml index bb4b7ea..19339de 100644 --- a/netbox/k8s/ingress.yaml +++ b/netbox/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`netbox.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: netbox-service port: 8080 diff --git a/netronome/k8s/ingress.yaml b/netronome/k8s/ingress.yaml index 0025d05..b8f7652 100644 --- a/netronome/k8s/ingress.yaml +++ b/netronome/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`nm.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: netronome-service port: 7575 diff --git a/nextcloud/k8s/routing/ingress.yaml b/nextcloud/k8s/routing/ingress.yaml index f9a7e84..3318161 100644 --- a/nextcloud/k8s/routing/ingress.yaml +++ b/nextcloud/k8s/routing/ingress.yaml @@ -12,8 +12,6 @@ spec: kind: Rule middlewares: - name: nextcloud-chain@file - - name: crowdsec-bouncer - namespace: crowdsec services: - name: nextcloud-apache port: 11000 @@ -32,8 +30,6 @@ spec: - match: Host(`nextcloud.workstation.internal`) || Host(`nextcloud.gigaforust.internal`) kind: Rule middlewares: - - name: crowdsec-bouncer - namespace: crowdsec - name: nextcloud-chain@file services: - name: nextcloud-apache diff --git a/portainer/k8s/ingress.yaml b/portainer/k8s/ingress.yaml index cc2283b..20e1c6a 100644 --- a/portainer/k8s/ingress.yaml +++ b/portainer/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`portainer.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: portainer-service port: 9000 diff --git a/prometheus-stack/k8s/crowdsec-alerts.yaml b/prometheus-stack/k8s/crowdsec-alerts.yaml new file mode 100644 index 0000000..2f77f81 --- /dev/null +++ b/prometheus-stack/k8s/crowdsec-alerts.yaml @@ -0,0 +1,41 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: crowdsec + namespace: prometheus + labels: + release: prometheus-stack +spec: + groups: + - name: crowdsec + rules: + - alert: CrowdsecFirewallBouncerStale + expr: | + absent(cs_lapi_bouncer_requests_total{bouncer="firewall-workstation"}) + or sum(rate(cs_lapi_bouncer_requests_total{bouncer="firewall-workstation"}[10m])) == 0 + for: 15m + labels: + severity: critical + annotations: + summary: "Firewall bouncer stopped pulling decisions" + description: "No LAPI pulls from firewall-workstation for 15 minutes. L3 enforcement is decaying: existing bans expire, new ones never land. Check the systemd unit on the node." + + - alert: CrowdsecDecisionsSpike + expr: | + sum(cs_active_decisions) - sum(cs_active_decisions offset 30m) > 20 + for: 5m + labels: + severity: warning + annotations: + summary: "Spike in active CrowdSec decisions" + description: "Active decisions grew by more than 20 in 30 minutes (current: {{ $value }}). Possible ban storm or self-ban - check cscli decisions list." + + - alert: CrowdsecLAPIDecisionErrors + expr: | + sum(rate(cs_lapi_decisions_ko_total[5m])) > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "CrowdSec LAPI decision errors" + description: "LAPI is failing decision lookups. Bouncers may be failing open. Check LAPI logs and DB health." diff --git a/searxng/k8s/ingress.yaml b/searxng/k8s/ingress.yaml index c75b27b..3c0d045 100644 --- a/searxng/k8s/ingress.yaml +++ b/searxng/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`s.forust.xyz`) || Host(`search.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: searxng-service port: 8080 diff --git a/termix/k8s/ingress.yaml b/termix/k8s/ingress.yaml index b6dcf28..f172d20 100644 --- a/termix/k8s/ingress.yaml +++ b/termix/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`termix.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: termix-service port: 8080 diff --git a/traefik/k8s/ingress.yaml b/traefik/k8s/ingress.yaml index f6e5a59..0b20f98 100644 --- a/traefik/k8s/ingress.yaml +++ b/traefik/k8s/ingress.yaml @@ -10,9 +10,6 @@ spec: routes: - match: Host(`traefik.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: api@internal kind: TraefikService diff --git a/traefik/k8s/traefik-values.yaml b/traefik/k8s/traefik-values.yaml index c5baaf4..655975d 100644 --- a/traefik/k8s/traefik-values.yaml +++ b/traefik/k8s/traefik-values.yaml @@ -68,7 +68,7 @@ providers: kubernetesCRD: enabled: true kubernetesGateway: - enabled: false + enabled: true file: enabled: false @@ -161,14 +161,13 @@ persistence: path: /data # No certificatesResolvers: public TLS comes from cert-manager. +# No plugins: L3 enforcement moved to the host firewall bouncer, +# nothing runs in the request path anymore. volumes: - name: traefik-dynamic mountPath: /etc/traefik/dynamic type: configMap - - name: crowdsec-bouncer-secrets - mountPath: /etc/traefik/secrets - type: secret additionalArguments: - "--providers.file.directory=/etc/traefik/dynamic" - "--providers.file.watch=true" @@ -177,12 +176,6 @@ additionalArguments: - "--entryPoints.websecure.forwardedHeaders.trustedIPs=173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,104.16.0.0/13,104.24.0.0/14,172.64.0.0/13,131.0.72.0/22,192.168.1.1,192.168.1.0/24,192.168.88.0/24,192.168.88.1" - "--entryPoints.web.forwardedHeaders.trustedIPs=173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,104.16.0.0/13,104.24.0.0/14,172.64.0.0/13,131.0.72.0/22,192.168.1.1,192.168.1.0/24,192.168.88.0/24,192.168.88.1" -experimental: - plugins: - crowdsec-bouncer: - moduleName: github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin - version: v1.3.3 - log: level: INFO accessLog: diff --git a/uptime-kuma/k8s/ingress.yaml b/uptime-kuma/k8s/ingress.yaml index 3f8f83e..1ffcf90 100644 --- a/uptime-kuma/k8s/ingress.yaml +++ b/uptime-kuma/k8s/ingress.yaml @@ -9,9 +9,6 @@ spec: routes: - match: Host(`uptime.forust.xyz`) kind: Rule - middlewares: - - name: crowdsec-bouncer - namespace: crowdsec services: - name: uptime-kuma-service port: 3001