feat(metrics): collect Headscale, NetBird, Gitea and Immich metrics

gitea/k8s/config.yaml

@@ -20,6 +20,8 @@ data:
 
   GITEA__mailer__ENABLED: "false"
 
+  GITEA__metrics__ENABLED: "true"
+
   # No code/issue search needed: bleve reindexes the whole issue index on
   # every pod restart (cron.rebuild_issue_indexer RUN_AT_START) and hammers
   # the rotational disk for an hour. "db" serves issue search from postgres.

gitea/k8s/gitea.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: gitea-service
   namespace: gitea
+  labels:
+    app: gitea
 spec:
   selector:
     app: gitea

gitea/k8s/ingress.yaml

@@ -7,7 +7,8 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`gitea.forust.xyz`) || Host(`git.forust.xyz`)
+    # Metrics are scraped directly through the cluster Service.
+    - match: (Host(`gitea.forust.xyz`) || Host(`git.forust.xyz`)) && !PathPrefix(`/metrics`)
       kind: Rule
       services:
         - name: gitea-service

gitea/k8s/servicemonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: gitea
+  namespace: gitea
+  labels:
+    release: prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      app: gitea
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

headscale/config/headscale.yaml.example

@@ -7,7 +7,7 @@
 # # Dev server_url
 # server_url: https://hs.dev_internal_domain.internal
 listen_addr: 0.0.0.0:8080
-metrics_listen_addr: 127.0.0.1:9090
+metrics_listen_addr: 0.0.0.0:9090
 grpc_listen_addr: 127.0.0.1:50443
 grpc_allow_insecure: false
 noise:

headscale/k8s/routing/external-service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: headscale-server-external
   namespace: headscale
+  labels:
+    app: headscale
 spec:
   ports:
     - port: 8080

headscale/k8s/vmservicescrape.yaml

@@ -0,0 +1,18 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: headscale
+  namespace: headscale
+  labels:
+    release: prometheus-stack
+spec:
+  # The external Service has a manually managed EndpointSlice, not Endpoints.
+  discoveryRole: endpointslice
+  selector:
+    matchLabels:
+      app: headscale
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

immich/k8s/config.yaml

@@ -6,6 +6,10 @@ metadata:
 data:
   TZ: "Europe/Bratislava"
 
+  IMMICH_TELEMETRY_INCLUDE: "all"
+  IMMICH_API_METRICS_PORT: "8081"
+  IMMICH_MICROSERVICES_METRICS_PORT: "8082"
+
   # The database in this namespace, not the shared one in the database
   # namespace: v3 needs VectorChord, and only the dedicated image carries it.
   DB_HOSTNAME: "immich-postgres"

immich/k8s/immich.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: immich-service
   namespace: immich
+  labels:
+    app: immich
 spec:
   selector:
     app: immich
@@ -10,6 +12,12 @@ spec:
     - name: http
       port: 2283
       targetPort: 2283
+    - name: api-metrics
+      port: 8081
+      targetPort: api-metrics
+    - name: worker-metrics
+      port: 8082
+      targetPort: worker-metrics
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -41,6 +49,10 @@ spec:
           ports:
             - name: http
               containerPort: 2283
+            - name: api-metrics
+              containerPort: 8081
+            - name: worker-metrics
+              containerPort: 8082
           volumeMounts:
             - name: immich-data
               mountPath: /data

immich/k8s/servicemonitor.yaml

@@ -0,0 +1,20 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: immich
+  namespace: immich
+  labels:
+    release: prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      app: immich
+  endpoints:
+    - port: api-metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s
+    - port: worker-metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

netbird/k8s/netbird.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: netbird-server-service
   namespace: netbird
+  labels:
+    app: netbird-server
 spec:
   selector:
     app: netbird-server
@@ -11,6 +13,10 @@ spec:
       name: http
       targetPort: 80
       protocol: TCP
+    - port: 9090
+      name: metrics
+      targetPort: metrics
+      protocol: TCP
     - port: 3478
       name: stun
       targetPort: 3478
@@ -59,6 +65,9 @@ spec:
             - containerPort: 80
               name: http
               protocol: TCP
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
             - containerPort: 3478
               name: stun
               protocol: UDP

netbird/k8s/servicemonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: netbird-server
+  namespace: netbird
+  labels:
+    release: prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      app: netbird-server
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

prometheus-stack/k8s/README.md

@@ -15,3 +15,29 @@ The VictoriaMetrics Operator chart and its CRDs are installed before the
 Kubernetes manifests by the normal deploy workflow. On a cluster where the
 operator CRDs are not installed yet, CI skips the server-side dry-run of the
 `VMAgent` resource; the deploy installs the chart before applying that resource.
+
+## Application metrics
+
+The application ServiceMonitors use a 30s interval and a 10s timeout:
+
+- Headscale: the external Service points to the Compose host on port 19090.
+  A VMServiceScrape uses EndpointSlice discovery for this manually managed target.
+  The Compose configuration must bind metrics to `0.0.0.0:9090`.
+- NetBird: the combined server exports `/metrics` on port 9090. The existing
+  `server.metricsPort` setting enables the listener.
+- Gitea: `GITEA__metrics__ENABLED` enables `/metrics` on the HTTP port. The public
+  ingress excludes this path. The monitor uses the internal Service directly.
+- Immich: `IMMICH_TELEMETRY_INCLUDE=all` enables API and worker metrics on ports
+  8081 and 8082. The monitor scrapes both ports on each server replica.
+
+Deploy through the existing CI and deploy workflow. Gitea and Immich reload their
+ConfigMap changes through Reloader. Check the VMAgent targets after deployment
+and query `up{scraper="victoria",namespace=~"netbird|gitea|immich|headscale"}` in
+VictoriaMetrics. All targets should report 1.
+
+For rollback, revert the application metrics changes, run CI, and deploy the
+revert. Remove the three application ServiceMonitors and the Headscale VMServiceScrape explicitly: the deployment
+workflow applies manifests and does not prune removed resources.
+
+For Headscale rollback, remove its VMServiceScrape and Service label, restore the
+previous Compose metrics bind address, and restart only the Headscale service.