style: prettier formatting for edu_master alerts and postgres readme

Deploy workflow uses git-tracked manifests, DISABLED flag and kustomize overlays; add webinar-checker metrics with ServiceMonitor and alerts; upgrade shared postgres to 17 with statuspage DB and probes/resources.

.gitea/workflows/ci.yaml

@@ -7,6 +7,10 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
 env:
   REGISTRY: gcr.forust.xyz
 
@@ -15,7 +19,7 @@ jobs:
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Check formatting with Prettier
         shell: bash
@@ -31,32 +35,24 @@ jobs:
             exit 0
           fi
 
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            node:22-alpine \
-            sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
+          prettier --check --ignore-unknown "${prettier_files[@]}"
 
   lint-ruff:
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Lint Python with Ruff
         shell: bash
         run: |
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/astral-sh/ruff:latest \
-            check .
+          ruff check .
 
   lint-yaml:
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Lint YAML syntax
         shell: bash
@@ -72,17 +68,13 @@ jobs:
             exit 0
           fi
 
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            cytopia/yamllint:latest \
-            -c .yamllint "${yaml_files[@]}"
+          yamllint -c .yamllint "${yaml_files[@]}"
 
   lint-dockerfiles:
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Lint Dockerfiles
         shell: bash
@@ -96,18 +88,13 @@ jobs:
             exit 0
           fi
 
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            --entrypoint hadolint \
-            hadolint/hadolint:latest-debian \
-            -c .hadolint.yaml "${dockerfiles[@]}"
+          hadolint -c .hadolint.yaml "${dockerfiles[@]}"
 
   validate:
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Validate Kubernetes manifests
         shell: bash
@@ -122,10 +109,7 @@ jobs:
             exit 0
           fi
 
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/yannh/kubeconform:latest \
+          kubeconform \
             -strict \
             -ignore-missing-schemas \
             -summary \
@@ -139,7 +123,7 @@ jobs:
       services: ${{ steps.services.outputs.services }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 0
 
@@ -230,7 +214,10 @@ jobs:
                 for tag in "${tags[@]}"; do
                   build_args+=(-t "${image}:${tag}")
                 done
-                docker build "${build_args[@]}" dtek_notif
+                docker build \
+                  --cache-from "type=registry,ref=${image}:buildcache" \
+                  --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                  "${build_args[@]}" dtek_notif
                 for tag in "${tags[@]}"; do
                   docker push "${image}:${tag}"
                 done
@@ -250,7 +237,10 @@ jobs:
                 for tag in "${tags[@]}"; do
                   build_args+=(-t "${image}:${tag}")
                 done
-                docker build "${build_args[@]}" errorpages
+                docker build \
+                  --cache-from "type=registry,ref=${image}:buildcache" \
+                  --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                  "${build_args[@]}" errorpages
                 for tag in "${tags[@]}"; do
                   docker push "${image}:${tag}"
                 done
@@ -280,7 +270,10 @@ jobs:
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build "${build_args[@]}" "$context"
+                  docker build \
+                    --cache-from "type=registry,ref=${image}:buildcache" \
+                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                    "${build_args[@]}" "$context"
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
@@ -309,7 +302,10 @@ jobs:
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
+                  docker build \
+                    --cache-from "type=registry,ref=${image}:buildcache" \
+                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                    "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
@@ -340,7 +336,10 @@ jobs:
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build "${build_args[@]}" "$context"
+                  docker build \
+                    --cache-from "type=registry,ref=${image}:buildcache" \
+                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                    "${build_args[@]}" "$context"
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
@@ -348,23 +347,3 @@ jobs:
                 ;;
             esac
           done
-
-  deploy-userbot-panel:
-    needs: build
-    if: github.ref_name == 'main' && contains(needs.build.outputs.services, 'userbot')
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Apply and roll out userbot panel
-        shell: bash
-        run: |
-          kubectl apply -f userbot/k8s/base/panel.yaml
-          kubectl get secret userbot-common-secrets -n default -o json \
-            | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
-            | kubectl apply -f -
-          # Keep legacy deployments (forust/anna) in sync with manifests; they have no replicas field, so apply leaves scaling to the user manager only.
-          kubectl apply -f userbot/k8s/base/userbots.yaml
-          kubectl rollout restart deployment/userbot-panel -n userbot
-          kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s

.gitea/workflows/deploy.yaml

@@ -1,6 +1,9 @@
 name: deploy
 
 on:
+  push:
+    branches:
+      - main
   workflow_dispatch:
 
 concurrency:
@@ -19,9 +22,6 @@ jobs:
           DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
           DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
           DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          # Set APPLY_PRUNE=true to enable kubectl apply --prune. Requires every
-          # manifest to carry label app.kubernetes.io/managed-by=homelab-deploy,
-          # otherwise previously applied resources get deleted on the next run.
           APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
         run: |
           set -euo pipefail
@@ -57,59 +57,170 @@ jobs:
           fi
 
           git -C "$repo" fetch origin main
-          git -C "$repo" reset --hard origin/main
 
-          # Runtime selection: a service is k8s-managed when $SERVICE/k8s/active
-          # exists. Otherwise it is compose-managed, and only k8s/routing/*
-          # manifests (external Services / EndpointSlices / ServersTransport /
-          # Ingresses that route to docker backends) are applied.
-          #   migrate:   touch SERVICE/k8s/active   (+ move routing files up)
-          #   rollback:  rm SERVICE/k8s/active
+          echo "== Workstation state =="
+          echo "  local:  $(git -C "$repo" rev-parse --short HEAD)"
+          echo "  remote: $(git -C "$repo" rev-parse --short origin/main)"
+
+          if [ -n "$(git -C "$repo" status --porcelain --untracked-files=no)" ]; then
+            echo "ERROR: workstation has local tracked modifications, refusing reset:"
+            git -C "$repo" status --porcelain --untracked-files=no
+            git -C "$repo" diff --stat
+            exit 1
+          fi
+
+          git -C "$repo" reset --hard origin/main
+          cd "$repo"
+
+          is_disabled() {
+            local target="$1"
+            if [ -f "$target" ]; then
+              target="$(dirname "$target")"
+            fi
+            while true; do
+              if [ -f "$target/DISABLED" ]; then
+                return 0
+              fi
+              if [ "$target" = "$repo" ]; then
+                break
+              fi
+              target="$(dirname "$target")"
+              case "$target" in
+                "$repo"/*) ;;
+                *) break ;;
+              esac
+            done
+            return 1
+          }
+
           collect_k8s() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              ! -path '*/routing/*' ! -path '*/overlays/*' \
-              ! -name 'kustomization.y*ml' ! -name '*.example.y*ml' \
-              ! -name '*values.y*ml' ! -name 'patch-*.y*ml' \
+            git ls-files -- "$1" \
+              | grep -E '\.ya?ml$' \
+              | grep -Ev '/routing/|/overlays/' \
+              | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$' \
+              | grep -Ev '(^|/)[^/]*secret[^/]*\.ya?ml$' \
               | sort
           }
 
           collect_k8s_inactive() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              \( -name 'namespace.y*ml' -o -path '*/routing/*' \) \
-              ! -path '*/overlays/*' ! -name '*.example.y*ml' \
-              | sort
+            collect_k8s "$1" \
+              | grep -E '(^|/)namespace\.ya?ml$|/routing/'
           }
 
-          mapfile -t compose_stacks < <(
-            find "$repo" -type f \( -name 'compose.yaml' -o -name 'compose.yml' \) | sort
+          kustomize_overlay() {
+            if [ -f "$1/overlays/prod/kustomization.yaml" ]; then
+              echo "$1/overlays/prod"
+            elif [ -f "$1/base/kustomization.yaml" ]; then
+              echo "$1/base"
+            fi
+          }
+
+          mapfile -t k8s_dirs < <(
+            git ls-files '*.yaml' '*.yml' \
+              | grep -E '(^|/)k8s/' \
+              | sed -E 's#((^|.*/)k8s)/.*#\1#' \
+              | sort -u
           )
 
-          mapfile -t k8s_manifests < <(
-            for kd in $(find "$repo" -type d -name k8s ! -path '*/.git/*' | sort); do
+          k8s_manifests=()
+          kustomize_apps=()
+          for kd_rel in "${k8s_dirs[@]}"; do
+            kd="$repo/$kd_rel"
+            if is_disabled "$kd"; then
+              echo "skip (DISABLED): $kd_rel"
+              continue
+            fi
             if [ -f "$kd/active" ]; then
-                collect_k8s "$kd"
+              overlay="$(kustomize_overlay "$kd" || true)"
+              if [ -n "${overlay:-}" ]; then
+                echo "kustomize app: ${overlay#$repo/}"
+                kustomize_apps+=("$overlay")
               else
-                collect_k8s_inactive "$kd"
+                while IFS= read -r f; do
+                  [ -n "$f" ] && k8s_manifests+=("$repo/$f")
+                done < <(collect_k8s "$kd_rel" || true)
+              fi
+            else
+              while IFS= read -r f; do
+                [ -n "$f" ] && k8s_manifests+=("$repo/$f")
+              done < <(collect_k8s_inactive "$kd_rel" || true)
             fi
           done
+
+          mapfile -t compose_rel < <(
+            git ls-files '*/compose.yaml' '*/compose.yml' compose.yaml compose.yml | sort
           )
 
+          compose_stacks=()
+          for cf_rel in "${compose_rel[@]}"; do
+            cf="$repo/$cf_rel"
+            if is_disabled "$cf"; then
+              echo "skip (DISABLED): $cf_rel"
+              continue
+            fi
+            if [ -f "$(dirname "$cf")/k8s/active" ]; then
+              echo "skip (k8s-managed): $cf_rel"
+              continue
+            fi
+            compose_stacks+=("$cf")
+          done
+
           echo "== Validate compose stacks =="
           for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
             echo "  config: $cf"
             docker compose -f "$cf" config --quiet
           done
 
-          echo "== Validate k8s manifests (kubectl dry-run) =="
+          echo "== Validate k8s manifests (kubectl dry-run=client) =="
           for m in "${k8s_manifests[@]}"; do
             echo "  apply --dry-run=client $m"
             kubectl apply --dry-run=client -f "$m" >/dev/null
           done
+          for k in "${kustomize_apps[@]}"; do
+            echo "  apply -k --dry-run=client $k"
+            kubectl apply -k "$k" --dry-run=client >/dev/null
+          done
+
+          echo "== Validate k8s manifests (kubectl dry-run=server) =="
+          for m in "${k8s_manifests[@]}"; do
+            echo "  apply --dry-run=server $m"
+            kubectl apply --dry-run=server -f "$m" >/dev/null
+          done
+          for k in "${kustomize_apps[@]}"; do
+            echo "  apply -k --dry-run=server $k"
+            kubectl apply -k "$k" --dry-run=server >/dev/null
+          done
+
+          echo "== Checking referenced Secrets exist =="
+          echo "  (deploy never applies *secret*.yaml; create missing ones from the laptop)"
+          ref_secrets=()
+          if [ "${#k8s_manifests[@]}" -gt 0 ]; then
+            while IFS= read -r s; do
+              [ -n "$s" ] && ref_secrets+=("$s")
+            done < <(
+              {
+                grep -h -A1 -E 'secretRef:|secretKeyRef:' "${k8s_manifests[@]}" 2>/dev/null || true
+                grep -h -E 'secretName:' "${k8s_manifests[@]}" 2>/dev/null || true
+              } | grep -E 'name:' | sed -E 's/.*name:[[:space:]]*//' | tr -d '"'"'"' "'"'" | sed -E 's/[[:space:]]*#.*//' | awk 'NF' | sort -u || true
+            )
+          fi
+          missing_secrets=()
+          all_secrets="$(kubectl get secrets -A --no-headers -o custom-columns=:metadata.name 2>/dev/null || true)"
+          for s in "${ref_secrets[@]}"; do
+            if printf '%s\n' "$all_secrets" | grep -qx "$s"; then
+              echo "  ok: $s"
+            else
+              echo "  MISSING: $s"
+              missing_secrets+=("$s")
+            fi
+          done
+          if [ "${#missing_secrets[@]}" -gt 0 ]; then
+            echo "ERROR: ${#missing_secrets[@]} referenced Secret(s) not found in the cluster:"
+            printf '  - %s\n' "${missing_secrets[@]}"
+            echo "Create them manually from the laptop, e.g.:"
+            echo "  kubectl apply -f SERVICE/k8s/secrets.yaml  # see SERVICE/k8s/secrets.yaml.example"
+            exit 1
+          fi
 
           echo "== Applying Kubernetes manifests =="
           ns_files=()
@@ -130,19 +241,63 @@ jobs:
             echo "  namespaces first: ${ns_files[*]}"
             kubectl apply -f "${ns_files[@]}"
           fi
+          if [ -f "$repo/prometheus-stack/k8s/active" ] && ! is_disabled "$repo/prometheus-stack/k8s"; then
+            if [ ! -f "$repo/prometheus-stack/k8s/grafana-values.yaml" ]; then
+              echo "ERROR: prometheus-stack/k8s/grafana-values.yaml (gitignored) missing on workstation, restore it first."
+              exit 1
+            fi
+            echo "== Upgrading kube-prometheus-stack =="
+            helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
+              --namespace prometheus \
+              --version 86.2.3 \
+              --values "$repo/prometheus-stack/k8s/grafana-values.yaml" \
+              --wait --timeout 10m
+          fi
+          if [ -f "$repo/loki/k8s/active" ] && ! is_disabled "$repo/loki/k8s"; then
+            echo "== Upgrading loki/alloy =="
+            helm repo add grafana https://grafana.github.io/helm-charts >/dev/null 2>&1 || true
+            helm repo update grafana >/dev/null 2>&1 || true
+            helm upgrade --install loki grafana/loki \
+              --version 7.3.0 \
+              --namespace prometheus \
+              --values "$repo/loki/k8s/loki-values.yaml" \
+              --wait --timeout 10m
+            helm upgrade --install alloy grafana/alloy \
+              --version 1.12.1 \
+              --namespace prometheus \
+              --values "$repo/loki/k8s/alloy-values.yaml" \
+              --wait --timeout 10m
+          fi
+
           if [ "${#other_files[@]}" -gt 0 ]; then
             echo "  resources: ${other_files[*]}"
             kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
           fi
 
+          for k in "${kustomize_apps[@]}"; do
+            echo "== Applying kustomize app: ${k#$repo/} =="
+            kubectl apply -k "$k"
+          done
+
+          if [ -f "$repo/userbot/k8s/active" ] && ! is_disabled "$repo/userbot"; then
+            echo "== userbot panel hook =="
+            if kubectl get secret userbot-common-secrets -n userbot >/dev/null 2>&1; then
+              echo "  userbot-common-secrets already present in userbot ns, not touching"
+            elif kubectl get secret userbot-common-secrets -n default >/dev/null 2>&1; then
+              echo "  bootstrapping userbot-common-secrets into userbot ns"
+              kubectl get secret userbot-common-secrets -n default -o json \
+                | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
+                | kubectl apply -f -
+            else
+              echo "  WARNING: userbot-common-secrets missing in both default and userbot ns; create it manually from the laptop"
+            fi
+            kubectl rollout restart deployment/userbot-panel -n userbot
+            kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s
+          fi
+
           echo "== Redeploying docker compose stacks =="
           for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  compose: $dir"
+            echo "  compose: $cf"
             if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
               docker compose -f "$cf" build
               docker compose -f "$cf" push

.gitea/workflows/renovate-ci.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Validate Renovate Compose draft
         shell: bash
@@ -48,5 +48,5 @@ jobs:
           docker run --rm \
             -v "$PWD:/work" \
             -w /work \
-            renovate/renovate:44.83.2 \
+            renovate/renovate:44.103.0 \
             renovate-config-validator renovate.json

.gitea/workflows/renovate-run.yaml

@@ -0,0 +1,69 @@
+name: renovate-run
+
+on:
+  workflow_dispatch:
+    inputs:
+      repositories:
+        description: "Repositories to scan (comma-separated)"
+        required: false
+        default: "forust/homelab"
+      log_level:
+        description: "Renovate log level"
+        required: false
+        default: "info"
+        type: choice
+        options:
+          - info
+          - debug
+      dry_run:
+        description: "Plan only, do not open or update PRs"
+        required: false
+        default: false
+        type: boolean
+
+concurrency:
+  group: renovate-run
+  cancel-in-progress: false
+
+jobs:
+  run-renovate:
+    runs-on: [self-hosted, linux, arch, homelab]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Validate Renovate config
+        shell: bash
+        run: |
+          set -euo pipefail
+          docker run --rm \
+            -v "$PWD/renovate/config.js:/opt/renovate/config.js:ro" \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/config.js \
+            renovate/renovate:44.103.0 \
+            renovate-config-validator
+
+      - name: Run Renovate
+        shell: bash
+        env:
+          RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
+          RENOVATE_GITHUB_COM_TOKEN: ${{ secrets.RENOVATE_GITHUB_COM_TOKEN }}
+          RENOVATE_REPOSITORIES: ${{ inputs.repositories }}
+          RENOVATE_DRY_RUN: ${{ inputs.dry_run && 'full' || '' }}
+          LOG_LEVEL: ${{ inputs.log_level }}
+        run: |
+          set -euo pipefail
+
+          : "${RENOVATE_TOKEN:?missing RENOVATE_TOKEN secret — add a renovate-bot PAT in repo/org Actions secrets}"
+
+          docker run --rm \
+            -v "$PWD/renovate/config.js:/opt/renovate/config.js:ro" \
+            -e RENOVATE_PLATFORM=gitea \
+            -e RENOVATE_ENDPOINT=https://gitea.forust.xyz/api/v1 \
+            -e RENOVATE_TOKEN="$RENOVATE_TOKEN" \
+            -e RENOVATE_GITHUB_COM_TOKEN="${RENOVATE_GITHUB_COM_TOKEN:-}" \
+            -e RENOVATE_REPOSITORIES="${RENOVATE_REPOSITORIES:-forust/homelab}" \
+            -e RENOVATE_DRY_RUN="${RENOVATE_DRY_RUN:-}" \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/config.js \
+            -e RENOVATE_BASE_DIR=/tmp/renovate \
+            -e LOG_LEVEL="${LOG_LEVEL:-info}" \
+            renovate/renovate:44.103.0

.github/workflows/ci.yaml

@@ -1,370 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches:
-      - "**"
-  pull_request:
-  workflow_dispatch:
-
-env:
-  REGISTRY: gcr.forust.xyz
-
-jobs:
-  lint-prettier:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Check formatting with Prettier
-        shell: bash
-        run: |
-          mapfile -t prettier_files < <(
-            git ls-files \
-              | grep -E '\.(md|json|ya?ml|html|css)$' \
-              | grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
-          )
-
-          if [ "${#prettier_files[@]}" -eq 0 ]; then
-            echo "No Prettier-managed files found."
-            exit 0
-          fi
-
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            node:22-alpine \
-            sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
-
-  lint-ruff:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Lint Python with Ruff
-        shell: bash
-        run: |
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/astral-sh/ruff:latest \
-            check .
-
-  lint-yaml:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Lint YAML syntax
-        shell: bash
-        run: |
-          mapfile -t yaml_files < <(
-            git ls-files '*.yaml' '*.yml' \
-              ':!node_modules/**' \
-              ':!**/.venv/**'
-          )
-
-          if [ "${#yaml_files[@]}" -eq 0 ]; then
-            echo "No YAML files found."
-            exit 0
-          fi
-
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            cytopia/yamllint:latest \
-            -c .yamllint "${yaml_files[@]}"
-
-  lint-dockerfiles:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Lint Dockerfiles
-        shell: bash
-        run: |
-          mapfile -t dockerfiles < <(
-            git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
-          )
-
-          if [ "${#dockerfiles[@]}" -eq 0 ]; then
-            echo "No Dockerfiles found."
-            exit 0
-          fi
-
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            --entrypoint hadolint \
-            hadolint/hadolint:latest-debian \
-            -c .hadolint.yaml "${dockerfiles[@]}"
-
-  validate:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Validate Kubernetes manifests
-        shell: bash
-        run: |
-          mapfile -t manifests < <(
-            git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
-              | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
-          )
-
-          if [ "${#manifests[@]}" -eq 0 ]; then
-            echo "No Kubernetes manifests found."
-            exit 0
-          fi
-
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/yannh/kubeconform:latest \
-            -strict \
-            -ignore-missing-schemas \
-            -summary \
-            "${manifests[@]}"
-
-  build:
-    needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
-    if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
-    runs-on: [self-hosted, linux, arch, homelab]
-    outputs:
-      services: ${{ steps.services.outputs.services }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Detect changed docker-built services
-        id: services
-        shell: bash
-        run: |
-          base="${{ github.event.before }}"
-          if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
-            base="$(git rev-list --max-parents=0 HEAD)"
-          fi
-
-          mapfile -t changed_files < <(git diff --name-only "$base" "${GITHUB_SHA}")
-
-          services=()
-
-          add_service() {
-            local name="$1"
-            local seen=0
-            for existing in "${services[@]}"; do
-              if [ "$existing" = "$name" ]; then
-                seen=1
-                break
-              fi
-            done
-            if [ "$seen" -eq 0 ]; then
-              services+=("$name")
-            fi
-          }
-
-          for file in "${changed_files[@]}"; do
-            case "$file" in
-              dtek_notif/*)
-                add_service dtek_notif
-                ;;
-              errorpages/*)
-                add_service errorpages
-                ;;
-              userbot/*)
-                add_service userbot
-                ;;
-              homepages/*)
-                add_service homepages
-                ;;
-              edu_master/phpsessid-bot/*|edu_master/webinar-checker/*|edu_master/compose.yaml)
-                add_service edu_master
-                ;;
-            esac
-          done
-
-          if [ "${#services[@]}" -eq 0 ]; then
-            echo "No docker-built services changed."
-            echo "services=" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          printf '%s\n' "${services[@]}" | tee /tmp/services.txt
-          echo "services=$(paste -sd, /tmp/services.txt)" >> "$GITHUB_OUTPUT"
-
-      - name: Log in to registry
-        if: steps.services.outputs.services != ''
-        shell: bash
-        run: |
-          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-            -u "${{ secrets.REGISTRY_USERNAME }}" \
-            --password-stdin
-
-      - name: Build and push changed images
-        if: steps.services.outputs.services != ''
-        shell: bash
-        run: |
-          IFS=, read -r -a services <<< "${{ steps.services.outputs.services }}"
-
-          for service in "${services[@]}"; do
-            case "$service" in
-              dtek_notif)
-                image="${REGISTRY}/forust/dtek-notif"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
-                build_args=()
-                for tag in "${tags[@]}"; do
-                  build_args+=(-t "${image}:${tag}")
-                done
-                docker build "${build_args[@]}" dtek_notif
-                for tag in "${tags[@]}"; do
-                  docker push "${image}:${tag}"
-                done
-                ;;
-              errorpages)
-                image="${REGISTRY}/forust/error-pages"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
-                build_args=()
-                for tag in "${tags[@]}"; do
-                  build_args+=(-t "${image}:${tag}")
-                done
-                docker build "${build_args[@]}" errorpages
-                for tag in "${tags[@]}"; do
-                  docker push "${image}:${tag}"
-                done
-                ;;
-              userbot)
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
-                for target in runtime panel; do
-                  case "$target" in
-                    runtime)
-                      context="userbot"
-                      image="${REGISTRY}/forust/userbot"
-                      ;;
-                    panel)
-                      context="userbot/panel"
-                      image="${REGISTRY}/forust/userbot-panel"
-                      ;;
-                  esac
-                  build_args=()
-                  for tag in "${tags[@]}"; do
-                    build_args+=(-t "${image}:${tag}")
-                  done
-                  docker build "${build_args[@]}" "$context"
-                  for tag in "${tags[@]}"; do
-                    docker push "${image}:${tag}"
-                  done
-                done
-                ;;
-              homepages)
-                for service in forust xdfnx; do
-                  case "$service" in
-                    forust)
-                      image="${REGISTRY}/forust/forust-homepage"
-                      ;;
-                    xdfnx)
-                      image="${REGISTRY}/forust/xdfnx-homepage"
-                      ;;
-                  esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
-                  build_args=()
-                  for tag in "${tags[@]}"; do
-                    build_args+=(-t "${image}:${tag}")
-                  done
-                  docker build "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
-                  for tag in "${tags[@]}"; do
-                    docker push "${image}:${tag}"
-                  done
-                done
-                ;;
-              edu_master)
-                for service in session-keeper webinar-checker; do
-                  case "$service" in
-                    session-keeper)
-                      context="edu_master/phpsessid-bot"
-                      image="${REGISTRY}/forust/session-keeper"
-                      ;;
-                    webinar-checker)
-                      context="edu_master/webinar-checker"
-                      image="${REGISTRY}/forust/webinar-checker"
-                      ;;
-                  esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
-                  build_args=()
-                  for tag in "${tags[@]}"; do
-                    build_args+=(-t "${image}:${tag}")
-                  done
-                  docker build "${build_args[@]}" "$context"
-                  for tag in "${tags[@]}"; do
-                    docker push "${image}:${tag}"
-                  done
-                done
-                ;;
-            esac
-          done
-
-  deploy-userbot-panel:
-    needs: build
-    if: github.ref_name == 'main' && contains(needs.build.outputs.services, 'userbot')
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Apply and roll out userbot panel
-        shell: bash
-        run: |
-          kubectl apply -f userbot/k8s/base/panel.yaml
-          kubectl get secret userbot-common-secrets -n default -o json \
-            | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
-            | kubectl apply -f -
-          # Keep legacy deployments (forust/anna) in sync with manifests; they have no replicas field, so apply leaves scaling to the user manager only.
-          kubectl apply -f userbot/k8s/base/userbots.yaml
-          kubectl rollout restart deployment/userbot-panel -n userbot
-          kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s

.github/workflows/deploy.yaml

@@ -1,161 +0,0 @@
-name: deploy
-
-on:
-  workflow_dispatch:
-
-concurrency:
-  group: deploy-main
-  cancel-in-progress: false
-
-jobs:
-  redeploy:
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    steps:
-      - name: Redeploy workstation
-        shell: bash
-        env:
-          DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
-          DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
-          DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
-          DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
-          DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          # Set APPLY_PRUNE=true to enable kubectl apply --prune. Requires every
-          # manifest to carry label app.kubernetes.io/managed-by=homelab-deploy,
-          # otherwise previously applied resources get deleted on the next run.
-          APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
-        run: |
-          set -euo pipefail
-
-          : "${DEPLOY_HOST:?missing DEPLOY_HOST}"
-          : "${DEPLOY_USER:?missing DEPLOY_USER}"
-          : "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
-
-          deploy_port="${DEPLOY_PORT:-22}"
-          deploy_path="${DEPLOY_PATH:-/srv/homelab}"
-
-          ssh_key="$RUNNER_TEMP/deploy_key"
-          mkdir -p "$RUNNER_TEMP"
-          printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
-          chmod 600 "$ssh_key"
-
-          ssh_opts=(
-            -i "$ssh_key"
-            -p "$deploy_port"
-            -o BatchMode=yes
-            -o StrictHostKeyChecking=accept-new
-          )
-
-          ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
-            "DEPLOY_PATH=$(printf '%q' \"$deploy_path\") APPLY_PRUNE=$(printf '%q' \"${APPLY_PRUNE:-false}\") bash -se" <<'EOF'
-          set -euo pipefail
-
-          repo="${DEPLOY_PATH:-/srv/homelab}"
-
-          if [ ! -d "$repo/.git" ]; then
-            echo "Repository not found at $repo"
-            exit 1
-          fi
-
-          git -C "$repo" fetch origin main
-          git -C "$repo" reset --hard origin/main
-
-          # Runtime selection: a service is k8s-managed when $SERVICE/k8s/active
-          # exists. Otherwise it is compose-managed, and only k8s/routing/*
-          # manifests (external Services / EndpointSlices / ServersTransport /
-          # Ingresses that route to docker backends) are applied.
-          #   migrate:   touch SERVICE/k8s/active   (+ move routing files up)
-          #   rollback:  rm SERVICE/k8s/active
-          collect_k8s() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              ! -path '*/routing/*' ! -path '*/overlays/*' \
-              ! -name 'kustomization.y*ml' ! -name '*.example.y*ml' \
-              ! -name '*values.y*ml' ! -name 'patch-*.y*ml' \
-              | sort
-          }
-
-          collect_k8s_inactive() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              \( -name 'namespace.y*ml' -o -path '*/routing/*' \) \
-              ! -path '*/overlays/*' ! -name '*.example.y*ml' \
-              | sort
-          }
-
-          mapfile -t compose_stacks < <(
-            find "$repo" -type f \( -name 'compose.yaml' -o -name 'compose.yml' \) | sort
-          )
-
-          mapfile -t k8s_manifests < <(
-            for kd in $(find "$repo" -type d -name k8s ! -path '*/.git/*' | sort); do
-              if [ -f "$kd/active" ]; then
-                collect_k8s "$kd"
-              else
-                collect_k8s_inactive "$kd"
-              fi
-            done
-          )
-
-          echo "== Validate compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  config: $cf"
-            docker compose -f "$cf" config --quiet
-          done
-
-          echo "== Validate k8s manifests (kubectl dry-run) =="
-          for m in "${k8s_manifests[@]}"; do
-            echo "  apply --dry-run=client $m"
-            kubectl apply --dry-run=client -f "$m" >/dev/null
-          done
-
-          echo "== Applying Kubernetes manifests =="
-          ns_files=()
-          other_files=()
-          for m in "${k8s_manifests[@]}"; do
-            case "$m" in
-              */namespace.y?ml) ns_files+=("$m") ;;
-              *) other_files+=("$m") ;;
-            esac
-          done
-
-          prune_opts=()
-          if [ "${APPLY_PRUNE:-false}" = "true" ]; then
-            prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
-          fi
-
-          if [ "${#ns_files[@]}" -gt 0 ]; then
-            echo "  namespaces first: ${ns_files[*]}"
-            kubectl apply -f "${ns_files[@]}"
-          fi
-          if [ -f "$repo/prometheus-stack/k8s/active" ]; then
-            echo "== Upgrading kube-prometheus-stack =="
-            helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
-              --namespace prometheus \
-              --version 86.2.3 \
-              --values "$repo/prometheus-stack/k8s/grafana-values.yaml" \
-              --wait
-          fi
-
-          if [ "${#other_files[@]}" -gt 0 ]; then
-            echo "  resources: ${other_files[*]}"
-            kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
-          fi
-
-          echo "== Redeploying docker compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  compose: $dir"
-            if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
-              docker compose -f "$cf" build
-              docker compose -f "$cf" push
-            fi
-            docker compose -f "$cf" up -d --pull always --remove-orphans
-          done
-          EOF

.gitignore

@@ -104,6 +104,10 @@ temp/*
 # kubernetes
 */k8s/*secret*
 !*/k8s/*secret*.example
+**/k8s/*secret*
+!**/k8s/*secret*.example
+# Local-only tweaks, not for upstream
+prometheus-stack/k8s/grafana-values.yaml
 traefik/k8s/local-tls.yaml
 converters/k8s/config.yaml
 convertx/k8s/config.yaml

adguardhome/k8s/adguard.yaml

@@ -62,6 +62,8 @@ spec:
     metadata:
       labels:
         app: adguard
+      annotations:
+        reloader.stakater.com/auto: "true"
     spec:
       containers:
         - name: adguard

adguardhome/k8s/certificate.yaml

@@ -0,0 +1,12 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: adguard-certs
+  namespace: adguard
+spec:
+  secretName: adguard-certs
+  dnsNames:
+    - dns.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer

adguardhome/k8s/ingress.yaml

@@ -7,7 +7,7 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`adguard.forust.xyz`) || Host(`dns.forust.xyz`)
+    - match: Host(`dns.forust.xyz`)
       kind: Rule
       middlewares:
         - name: crowdsec-bouncer
@@ -15,13 +15,13 @@ spec:
       services:
         - name: adguard-service
           port: 3000
-    - match: (Host(`adguard.forust.xyz`) || Host(`dns.forust.xyz`)) && PathPrefix(`/dns-query`)
+    - match: (Host(`dns.forust.xyz`)) && PathPrefix(`/dns-query`)
       kind: Rule
       services:
         - name: adguard-service
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: adguard-certs
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -42,3 +42,5 @@ spec:
       services:
         - name: adguard-service
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls

adguardhome/k8s/internal-certificate.yaml

@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: adguard
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

authentik/k8s/authentik.yaml

@@ -41,7 +41,7 @@ spec:
     spec:
       containers:
         - name: authentik-server
-          image: ghcr.io/goauthentik/server:2026.8.2
+          image: ghcr.io/goauthentik/server:2026.8.3
           args: ["server"]
           envFrom:
             - configMapRef:
@@ -75,7 +75,7 @@ spec:
     spec:
       containers:
         - name: authentik-worker
-          image: ghcr.io/goauthentik/server:2026.8.2
+          image: ghcr.io/goauthentik/server:2026.8.3
           args: ["worker"]
           securityContext:
             runAsUser: 0

authentik/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: authentik-prod-tls
+  namespace: authentik
+spec:
+  secretName: authentik-prod-tls
+  dnsNames:
+    - auth.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: authentik
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

authentik/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: authentik-server-service
           port: 9000
   tls:
-    certResolver: letsencrypt
+    secretName: authentik-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -32,3 +32,5 @@ spec:
       services:
         - name: authentik-server-service
           port: 9000
+  tls:
+    secretName: internal-wildcard-tls

cert-manager/k8s/cert-manager-values.yaml

@@ -0,0 +1,9 @@
+crds:
+  enabled: true
+prometheus:
+  servicemonitor:
+    enabled: true
+    interval: 60s
+    scrapeTimeout: 30s
+    labels:
+      release: prometheus-stack

cert-manager/k8s/clusterissuer.yaml

@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    email: bobrovod@national.shitposting.agency
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: letsencrypt-staging-account-key
+    solvers:
+      - http01:
+          ingress:
+            class: traefik
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    email: bobrovod@national.shitposting.agency
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: letsencrypt-prod-account-key
+    solvers:
+      - http01:
+          ingress:
+            class: traefik

cert-manager/k8s/homelab-internal-root-ca.crt

@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFjCCAv6gAwIBAgIUetKpTfEDOn2985FFMu6G26itT+wwDQYJKoZIhvcNAQEN
+BQAwIzEhMB8GA1UEAxMYaG9tZWxhYiBpbnRlcm5hbCByb290IENBMB4XDTI2MDky
+MzEyNDA0N1oXDTM2MDkyMDEyNDA0N1owIzEhMB8GA1UEAxMYaG9tZWxhYiBpbnRl
+cm5hbCByb290IENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvmNP
+ZCOoD8NtNuYJKVXBlTPjX7D7sJCSK5neH7ZbYV5+lmUlEErY8Mik7j37V5k5NfpF
+Ig85pOjP7RckTPz5V6ek3yaN40s4AL053sN5ZPauDVYjalaEHTgj5sEMqlLACQWI
+yZmJOZspZykae8dIpQnqCoFpRT4FurJ78v4a0ylnFVLMQn/lyCHedwTjkEdtYWYr
+ccJy8vQwqkzs/rWvEH1lDqZhennLOrmcCjfonG7D/pruMn4z+6E28p4+ejkRrI6x
+luak3KnpT1XMeHtgU21hiRGaMDBchHMFgAhnY1qosymKenXvfTZItwgjZbwa1hJI
+GAiDm+jQDKMjzRZ3rH6Xfc0auUcykNz73PpNu1NGm78nndXwCXcXn1LFKNQJ+r1U
+sJiyAmUZmXVn4aM4OMf2F38k7wTYIKg7nRGaUkNeKDlNkjA4HvgWw+jwO1KmdHQ/
+mOem1rosDWHRK01wg+Gga9mQCnhNhxglg3t/UeSic6uOaRsvaz4qkzHq8MbCujVz
+DpKQjqdikYOAXZOs4KlBLWrS7NaK4NzfSD02pBUErh54ruJfY/bWz9KyXzBD/lQZ
+VUTKyvUVB0bkVHEdf1jJmX3H4IZRQSF5JPqOBotW6bJI5fEGNBvj9Zxy4nm2WWGz
+yyP3uWsQz8U/Wdx9nXZLHInTZBsvgLYtKUAWA30CAwEAAaNCMEAwDgYDVR0PAQH/
+BAQDAgKkMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEkKm2rxPaK6+O9WD80z
+BLC6F9QsMA0GCSqGSIb3DQEBDQUAA4ICAQAnFyHz97Umf5VIu+dKTJid7C73VugJ
+TIar/xJBs/4CxP+znBxhJjXygRoyIfzoVGWcB2ZSL//vL78Qlts79K/Imc9a4RFF
+wMvCxsRXAEQ4TpeWi3ophPNcs4rhsP+gQKQFtnyKP9519bqpfxp0bTqwOV2o18fn
+za7rlQViiEnNV58j7CVoM9+mJvVVfBEX1Km+GyJL9GadzbIQ7FxClVJZefCbft93
+zHVk9gDOw8ys1XGSR2OUCyCLinXO6mqS16CmBb2MAKXq/YyH7E0N8iotAPGtfA8V
+M/0ddy947rY0xCrtECfWwvGQpJS7NRv/Z9b2jCfXrI5LXmL2nfQRg0y9GE4Vjwr+
+WxtGU5jOeFt0jQ+xRzcgG0Op+qK3x55l5LSo2hOcOVYbiHxcHEJFgwNi1ADeBFwb
+q/HdysfURSOghqjIpMMAUabBp+DBUg2EUF7pIaUqbdqExFYcr9EYisEMiNsmKmN+
+8ZbcOeerFKDQj+t/R0bFXa7UBn2UWsjI8zlR74aa2kLDXwtyz/XlO/FlYm66eBFo
+2/eYUSeU+S4ej+wUAs/dvjF7f190/DUQGuwOTlLTahqWDztmhCk7qzbECu56CwKT
+E5Ect2P72UleYwdblkVOVd352AmiwEzdOaziIRrPh8uenEknH6JBYPO3mjk7cCg+
+GPGcNIBctjdXhg==
+-----END CERTIFICATE-----

cert-manager/k8s/internal-ca.yaml

@@ -0,0 +1,33 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned
+spec:
+  selfSigned: {}
+---
+# Homelab internal root CA (10y). Install the .crt on clients (see below).
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-ca-root
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: homelab internal root CA
+  duration: 87600h
+  renewBefore: 7200h
+  secretName: internal-ca-root
+  privateKey:
+    algorithm: RSA
+    size: 4096
+  issuerRef:
+    name: selfsigned
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: internal-ca
+spec:
+  ca:
+    secretName: internal-ca-root

cert-manager/k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager

checkmk/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: checkmk-prod-tls
+  namespace: checkmk
+spec:
+  secretName: checkmk-prod-tls
+  dnsNames:
+    - cmk.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: checkmk
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

checkmk/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: checkmk-service
           port: 5000
   tls:
-    certResolver: letsencrypt
+    secretName: checkmk-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRouteTCP
@@ -48,3 +48,5 @@ spec:
       services:
         - name: checkmk-service
           port: 5000
+  tls:
+    secretName: internal-wildcard-tls

cloudflared/k8s/.gitignore

@@ -0,0 +1 @@
+secret.yaml

cloudflared/k8s/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared
+  labels:
+    app: cloudflared
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cloudflared
+  template:
+    metadata:
+      labels:
+        app: cloudflared
+    spec:
+      containers:
+        - name: cloudflared
+          image: cloudflare/cloudflared:2026.9.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - tunnel
+            - --no-autoupdate
+            - run
+          env:
+            - name: TUNNEL_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: cloudflared-secrets
+                  key: TUNNEL_TOKEN
+          resources:
+            requests:
+              memory: "32Mi"
+              cpu: "30m"
+            limits:
+              memory: "128Mi"
+              cpu: "200m"

cloudflared/k8s/secret.yaml.example

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloudflared-secrets
+type: Opaque
+stringData:
+  TUNNEL_TOKEN: your_tunnel_token_here

converters/k8s/certificates.yaml

@@ -0,0 +1,42 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: convertx-prod-tls
+  namespace: converters
+spec:
+  secretName: convertx-prod-tls
+  dnsNames:
+    - forust.xyz
+    - www.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: bentopdf-prod-tls
+  namespace: converters
+spec:
+  secretName: bentopdf-prod-tls
+  dnsNames:
+    - pdf.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: converters
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

converters/k8s/ingress.yaml

@@ -14,7 +14,7 @@ spec:
         - name: convertx-service
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: convertx-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,6 +31,9 @@ spec:
       services:
         - name: convertx-service
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -47,7 +50,7 @@ spec:
         - name: bentopdf-service
           port: 8080
   tls:
-    certResolver: letsencrypt
+    secretName: bentopdf-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -63,3 +66,5 @@ spec:
       services:
         - name: bentopdf-service
           port: 8080
+  tls:
+    secretName: internal-wildcard-tls

crowdsec/k8s/crowdsec-values.yaml

@@ -1,57 +1,102 @@
 container_runtime: containerd
+
 agent:
+  acquisition: []
+  additionalAcquisition:
+    - labels:
+        type: traefik
+      limit: 1000
+      query: |
+        {namespace="traefik"}
+      source: loki
+      url: http://loki.prometheus.svc.cluster.local:3100/
+      wait_for_ready: 30s
   env:
     - name: COLLECTIONS
-      value: "crowdsecurity/traefik crowdsecurity/base-http-scenarios"
+      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
     - name: DISABLE_COLLECTIONS
-      value: "crowdsecurity/linux crowdsecurity/sshd"
-
-  acquisition:
-    - namespace: traefik
-      podName: "*traefik*"
-      program: traefik
-      poll_without_inotify: true
-
-  resources:
-    requests:
-      cpu: 50m
-      memory: 100Mi
-    limits:
-      cpu: 200m
-      memory: 500Mi
-
-lapi:
-  env:
-    - name: COLLECTIONS
-      value: "crowdsecurity/traefik crowdsecurity/base-http-scenarios"
-    - name: DISABLE_COLLECTIONS
-      value: "crowdsecurity/linux crowdsecurity/sshd"
-  service:
-    type: ClusterIP
-  persistentVolume:
-    data:
-      enabled: true
-      storageClassName: local-path-retain
-      size: 1Gi
-    config:
-      enabled: true
-      storageClassName: local-path-retain
-      size: 100Mi
-  storeLAPICscliCredentialsInSecret: true
-  resources:
-    requests:
-      cpu: 50m
-      memory: 150Mi
-    limits:
-      cpu: 400m
-      memory: 500Mi
-
+      value: crowdsecurity/sshd
   metrics:
     enabled: true
     serviceMonitor:
       additionalLabels:
         release: prometheus-stack
       enabled: true
-    interval: 30s
-    scrapeTimeout: 10s
-    namespace: prometheus
+  # Static machine identity: agent pods mount pre-created LAPI credentials
+  # (Secret crowdsec-agent-credentials, key local_api_credentials.yaml)
+  # at the exact path the agent entrypoint expects. Together with the
+  # patched register-init (enforced by janitor-cronjob.yaml) the agent
+  # never calls `cscli lapi register` in steady state, so pod names,
+  # restarts and reboots can no longer break it.
+  extraVolumes:
+    - name: static-creds
+      secret:
+        secretName: crowdsec-agent-credentials
+        items:
+          - key: local_api_credentials.yaml
+            path: local_api_credentials.yaml
+  extraVolumeMounts:
+    - name: static-creds
+      mountPath: /tmp_config/local_api_credentials.yaml
+      subPath: local_api_credentials.yaml
+      readOnly: true
+  resources:
+    limits:
+      cpu: 200m
+      memory: 500Mi
+    requests:
+      cpu: 50m
+      memory: 100Mi
+
+config:
+  parsers:
+    s02-enrich:
+      mobile-whitelist.yaml: |
+        name: forust/mobile-whitelist
+        description: "Whitelist SWAN/4ka mobile network"
+        whitelist:
+          reason: "Mobile IP whitelist"
+          cidr:
+            - "84.245.64.0/18"
+
+  postoverflows:
+    s01-whitelist:
+      home-dynamic-ip.yaml: |
+        name: forust/home-dynamic-ip
+        description: "Whitelist home dynamic IP"
+        whitelist:
+          reason: "Home dynamic IP"
+          expression:
+            - evt.Overflow.Alert.Source.IP in LookupHost("ddns.forust.xyz")
+
+lapi:
+  env:
+    - name: COLLECTIONS
+      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
+    - name: DISABLE_COLLECTIONS
+      value: crowdsecurity/linux crowdsecurity/sshd
+  metrics:
+    enabled: true
+    serviceMonitor:
+      additionalLabels:
+        release: prometheus-stack
+      enabled: true
+  persistentVolume:
+    config:
+      enabled: true
+      size: 100Mi
+      storageClassName: local-path-retain
+    data:
+      enabled: true
+      size: 1Gi
+      storageClassName: local-path-retain
+  resources:
+    limits:
+      cpu: 400m
+      memory: 500Mi
+    requests:
+      cpu: 50m
+      memory: 150Mi
+  service:
+    type: ClusterIP
+  storeLAPICscliCredentialsInSecret: true

crowdsec/k8s/grafana-dashboards.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+data:
+  crowdsec-overview.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph (old)\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"timeseries\",\n      \"name\": \"Time series\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 24,\n      \"panels\": [],\n      \"title\": \"Summary\",\n      \"type\": \"row\"\n    },\n    {\n      \"cacheTimeout\": null,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [\n            {\n              \"options\": {\n                \"match\": \"null\",\n                \"result\": {\n                  \"text\": \"N/A\"\n                }\n              },\n              \"type\": \"special\"\n            }\n          ],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"#E02F44\",\n                \"value\": null\n              },\n              {\n                \"color\": \"#E02F44\",\n                \"value\": 10\n              },\n              {\n                \"color\": \"#299c46\",\n                \"value\": 10\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 6,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"interval\": null,\n      \"links\": [],\n      \"maxDataPoints\": 100,\n      \"options\": {\n        \"colorMode\": \"background\",\n        \"graphMode\": \"none\",\n        \"justifyMode\": \"auto\",\n        \"orientation\": \"horizontal\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"text\": {},\n        \"textMode\": \"auto\"\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"count(cs_info)\",\n          \"interval\": \"\",\n          \"legendFormat\": \"\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"timeFrom\": null,\n      \"timeShift\": null,\n      \"title\": \"Running Crowdsec\",\n      \"transparent\": true,\n      \"type\": \"stat\"\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"decimals\": 1,\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"links\": []\n        },\n        \"overrides\": []\n      },\n      \"fill\": 1,\n      \"fillGradient\": 0,\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 18,\n        \"x\": 6,\n        \"y\": 1\n      },\n      \"hiddenSeries\": false,\n      \"id\": 8,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": false,\n        \"max\": false,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": \"total\",\n        \"sortDesc\": true,\n        \"total\": true,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"nullPointMode\": \"null\",\n      \"options\": {\n        \"alertThreshold\": true\n      },\n      \"percentage\": false,\n      \"pluginVersion\": \"8.1.2\",\n      \"pointradius\": 2,\n      \"points\": false,\n      \"renLine truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: "1"
+  name: crowdsec-crowdsec-overview
+  namespace: prometheus
+---
+apiVersion: v1
+data:
+  crowdsec-lapi-metrics.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915193937,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 10,\n      \"panels\": [],\n      \"title\": \"Agents\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              },\n              {\n                \"color\": \"red\",\n                \"value\": 80\n              }\n            ]\n          }\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"vertical\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"repeat\": \"query0\",\n      \"repeatDirection\": \"h\",\n      \"targets\": [\n        {\n          \"exemplar\": false,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\", instance=\\\"$lapi\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Agents Login\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 1\n      },\n      \"id\": 6,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"auto\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Heartbeat\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 12,\n      \"panels\": [],\n      \"title\": \"Decisions\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n  Line truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: "1"
+  name: crowdsec-crowdsec-lapi-metrics
+  namespace: prometheus
+---
+apiVersion: v1
+data:
+  crowdsec-insight.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"gauge\",\n      \"name\": \"Gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915159751,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 22,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"color\": {\n                \"mode\": \"thresholds\"\n              },\n              \"mappings\": [\n                {\n                  \"options\": {\n                    \"match\": \"null\",\n                    \"result\": {\n                      \"text\": \"N/A\"\n                    }\n                  },\n                  \"type\": \"special\"\n                }\n              ],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  },\n                  {\n                    \"color\": \"red\",\n                    \"value\": 80\n                  }\n                ]\n              },\n              \"unit\": \"dateTimeAsIso\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 2,\n            \"y\": 1\n          },\n          \"id\": 2,\n          \"interval\": null,\n          \"links\": [],\n          \"maxDataPoints\": 100,\n          \"options\": {\n            \"colorMode\": \"none\",\n            \"graphMode\": \"none\",\n            \"justifyMode\": \"auto\",\n            \"orientation\": \"horizontal\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"lastNotNull\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\n            \"text\": {},\n            \"textMode\": \"auto\"\n          },\n          \"pluginVersion\": \"8.1.2\",\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"(process_start_time_seconds{instance=\\\"$instance\\\"})*1000\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Up since\",\n          \"type\": \"stat\"\n        },\n        {\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"displayName\": \"\",\n              \"mappings\": [],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  }\n                ]\n              },\n              \"unit\": \"decbytes\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 7,\n            \"y\": 1\n          },\n          \"id\": 4,\n          \"options\": {\n            \"orientation\": \"auto\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"mean\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\Line truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: "1"
+  name: crowdsec-crowdsec-insight
+  namespace: prometheus

crowdsec/k8s/janitor-cronjob.yaml

@@ -0,0 +1,195 @@
+# CrowdSec self-healing: static machine identity + enforcement loops.
+#
+# Problem it fixes: the chart's agent init container runs
+#   `cscli lapi register --machine "$POD_NAME" ...`
+# unconditionally. Credentials live in an emptyDir, the machine row lives
+# in LAPI's persistent DB. Any init re-run for an already-known pod name
+# (kubelet restart, node reboot) dies with
+#   403 Forbidden: user '<pod>' already exist
+# and the DaemonSet pod sticks in Init forever. Every DS restart also
+# leaves an orphan machine row that is never cleaned.
+#
+# Design (name-independent):
+#   * Agent identity is a STATIC machine `crowdsec-agent-workstation`
+#     whose password lives in Secret `crowdsec-agent-credentials`
+#     (created once, manually - like all other secrets in this repo).
+#     The secret is mounted into agent pods at
+#     /tmp_config/local_api_credentials.yaml (see extraVolumeMounts in
+#     crowdsec-values.yaml), which is exactly the path the agent's main
+#     container copies into place at startup.
+#   * The DS init command is patched (strategic merge, by container name)
+#     to SKIP registration when that file exists, keeping the legacy
+#     register path only as fallback. Detection marker in the patched
+#     command: `[ -s /tmp_config`.
+#   * This CronJob enforces the desired state hourly, so recovery is
+#     automatic even after `helm upgrade` reverts the DS patch or the
+#     LAPI database is wiped:
+#       1. patch DS init if it still has the unconditional register
+#          (no-op otherwise - no restart churn);
+#       2. prune machines with no heartbeat for 2h (orphan hygiene);
+#       3. ensure the static machine exists, recreating it with the
+#          Secret password if missing (agent retry loops reconnect
+#          on their own - same name + same password);
+#       4. prune bouncer entries idle for 30d.
+#
+# Manual apply (crowdsec/k8s is NOT managed by deploy.yaml):
+#   kubectl apply -f crowdsec/k8s/janitor-cronjob.yaml
+# Force a run:
+#   kubectl create job -n crowdsec --from=cronjob/crowdsec-janitor janitor-now
+#
+# Helm upgrades: the janitor's strategic patch puts the DS field under
+# the `kubectl-patch` field manager, so a plain `helm upgrade` FAILS
+# with an SSA conflict on initContainers[].command. Procedure:
+#   1. revert init to chart state (kills the conflict):
+#        helm template crowdsec crowdsec/crowdsec --version <ver> \
+#          -n crowdsec -f crowdsec/k8s/crowdsec-values.yaml > /tmp/r.yaml
+#        python3 -c "import yaml,json; ..." # build revert patch from
+#        the rendered DaemonSet init command, then
+#        kubectl patch ds crowdsec-agent -n crowdsec \
+#          --type strategic -p "\$(cat /tmp/revert_patch.json)"
+#   2. helm upgrade --install crowdsec ... (no --force needed)
+#   3. janitor-now right away (upgrade reverts init; new pods would
+#      sit in Init until the next hourly run otherwise).
+#
+# One-time bootstrap (order matters):
+#   1. Create Secret + static machine (see commands in chat).
+#   2. Apply this file, trigger janitor-now, wait for agent 1/1.
+#   3. One-time orphan cleanup:
+#        kubectl exec -n crowdsec deploy/crowdsec-lapi -- \
+#          cscli machines prune --duration 1h --force
+#   4. Only then `helm upgrade` crowdsec with the extraVolumes values.
+#      Upgrade reverts the DS patch; trigger janitor-now right after it
+#      (otherwise new pods sit in Init until the next hourly run, then
+#      self-heal anyway).
+#
+# Password rotation: update the Secret, delete the machine
+# (`cscli machines delete crowdsec-agent-workstation`), trigger
+# janitor-now (recreates it), then `kubectl rollout restart
+# ds/crowdsec-agent -n crowdsec` (agent reads the file at startup only).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: ["apps"]
+    resources: ["daemonsets"]
+    verbs: ["get", "patch"]
+  # `kubectl exec deploy/<name>` resolves deploy -> replicaset -> pod,
+  # which needs read access to these (exec itself is pods/exec above).
+  - apiGroups: ["apps"]
+    resources: ["deployments", "replicasets"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+subjects:
+  - kind: ServiceAccount
+    name: crowdsec-janitor
+    namespace: crowdsec
+roleRef:
+  kind: Role
+  name: crowdsec-janitor
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+spec:
+  schedule: "17 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 300
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/part-of: crowdsec
+        spec:
+          serviceAccountName: crowdsec-janitor
+          restartPolicy: OnFailure
+          containers:
+            - name: janitor
+              # Same image the chart itself uses for registration jobs;
+              # IfNotPresent so it works while the node is offline
+              # (layer cached from the chart install).
+              image: alpine/kubectl:latest
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: AGENT_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: crowdsec-agent-credentials
+                      key: password
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  LAPI_EXEC="kubectl exec -n crowdsec deploy/crowdsec-lapi --"
+                  echo "== 1. enforce patched agent init =="
+                  CUR=$(kubectl get ds crowdsec-agent -n crowdsec \
+                    -o jsonpath='{.spec.template.spec.initContainers[0].command[2]}')
+                  case "$CUR" in
+                    *'-s /tmp_config'*)
+                      echo "init already patched"
+                      ;;
+                    *)
+                      echo "patching init"
+                      WAIT='until nc "$LAPI_HOST" "$LAPI_PORT" -z'
+                      WAIT="$WAIT; do echo waiting for lapi to start; sleep 5; done"
+                      LINK='ln -s /staging/etc/crowdsec /etc/crowdsec'
+                      REG='cscli lapi register --machine "$USERNAME"'
+                      REG="$REG -u \"\$LAPI_URL\" --token \"\$REGISTRATION_TOKEN\""
+                      CREDS=/tmp_config/local_api_credentials.yaml
+                      CMD="$WAIT; $LINK; [ -s $CREDS ] || {"
+                      CMD="$CMD $REG && cp"
+                      CMD="$CMD /etc/crowdsec/local_api_credentials.yaml $CREDS; }"
+                      ESC=$(printf '%s' "$CMD" | sed 's/"/\\"/g')
+                      PATCH='{"spec":{"template":{"spec":{"initContainers":'
+                      PATCH=$PATCH'[{"name":"wait-for-lapi-and-register",'
+                      PATCH=$PATCH'"command":["sh","-c","'$ESC'"]}]}}}}'
+                      kubectl patch ds crowdsec-agent -n crowdsec \
+                        --type strategic -p "$PATCH"
+                      ;;
+                  esac
+                  echo "== 2. prune orphan machines (no heartbeat for 2h) =="
+                  $LAPI_EXEC cscli machines prune --duration 2h --force
+                  echo "== 3. ensure static machine exists =="
+                  if $LAPI_EXEC cscli machines inspect \
+                    crowdsec-agent-workstation >/dev/null 2>&1; then
+                    echo "static machine present"
+                  else
+                    echo "recreating static machine"
+                    $LAPI_EXEC cscli machines add crowdsec-agent-workstation \
+                      --password "$AGENT_PASSWORD" --force
+                  fi
+                  echo "== 4. prune stale bouncers (no pull for 30d) =="
+                  $LAPI_EXEC cscli bouncers prune -d 720h --force

crowdsec/k8s/network-policy.yaml

@@ -25,3 +25,10 @@ spec:
       ports:
         - protocol: TCP
           port: 8080
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: prometheus
+      ports:
+        - protocol: TCP
+          port: 6060

dockmon/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: dockmon-prod-tls
+  namespace: dockmon
+spec:
+  secretName: dockmon-prod-tls
+  dnsNames:
+    - dockmon.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: dockmon
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

dockmon/k8s/ingress.yaml

@@ -26,7 +26,7 @@ spec:
           port: 443
           serversTransport: dockmon-transport
   tls:
-    certResolver: letsencrypt
+    secretName: dockmon-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -43,3 +43,5 @@ spec:
         - name: dockmon-service
           port: 443
           serversTransport: dockmon-transport
+  tls:
+    secretName: internal-wildcard-tls

downtify/compose.yaml

@@ -1,7 +1,7 @@
 services:
   downtify:
     container_name: downtify
-    image: ghcr.io/henriquesebastiao/downtify:2.12.0
+    image: ghcr.io/henriquesebastiao/downtify:2.13.0
     restart: unless-stopped
     # ports:
     #   - '7077:8000'

downtify/k8s/downtify.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: downtify
-          image: ghcr.io/henriquesebastiao/downtify:2.12.0
+          image: ghcr.io/henriquesebastiao/downtify:2.13.0
           ports:
             - containerPort: 8000
           volumeMounts:

downtify/k8s/ingress.yaml

@@ -17,7 +17,7 @@ spec:
         - name: downtify-service
           port: 8000
   tls:
-    certResolver: letsencrypt
+    secretName: downtify-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -33,3 +33,5 @@ spec:
       services:
         - name: downtify-service
           port: 8000
+  tls:
+    secretName: internal-wildcard-tls

edu_master/PLAYWRIGHT_VERSION

@@ -0,0 +1 @@
+1.56.0

edu_master/compose.yaml

@@ -11,7 +11,7 @@ services:
       retries: 5
 
   playwright-service:
-    image: mcr.microsoft.com/playwright:v1.63.0-jammy
+    image: mcr.microsoft.com/playwright:v1.56.0-jammy
     restart: unless-stopped
     command: npx -y playwright@1.56.0 run-server --port 3000 --path /ws
 

edu_master/k8s/alerts.yaml

@@ -0,0 +1,77 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: edu-master-webinar
+  namespace: edu-master
+  labels:
+    release: prometheus-stack
+spec:
+  groups:
+    - name: edu_master.webinar
+      rules:
+        # No successful webinar check for 5m (~2-3 missed 2-min checks).
+        # Catches: playwright hangs/timeouts, version skew, site changes, hung job.
+        - alert: WebinarCheckerNoSuccessfulCheck
+          expr: |
+            (time() - webinar_check_last_success_timestamp_seconds > 300)
+              and (webinar_check_last_run_timestamp_seconds > 0)
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker has no successful check for 5m"
+            description: "edu-master/webinar-checker: last successful webinar check was {{ $value | humanizeDuration }} ago. Checks are failing or hanging (see consecutive failures alert). Notifications about new webinars are NOT being sent."
+
+        # Fast path: 3 consecutive failures (~6+ min at 2-min interval).
+        - alert: WebinarCheckerConsecutiveFailures
+          expr: |
+            webinar_check_consecutive_failures >= 3
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker failing consecutively"
+            description: 'edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / playwright error / page error). Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
+
+        # Metrics endpoint not scraped for 10m: pod down, metrics server dead, or ServiceMonitor broken.
+        - alert: WebinarCheckerScrapeDown
+          expr: |
+            absent(webinar_check_last_run_timestamp_seconds) == 1
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker metrics missing"
+            description: "edu-master/webinar-checker: no metrics series for 10m. Pod may be down, metrics server dead, or ServiceMonitor/Service broken. Webinar checks are unobserved."
+
+        # EDU session lost: session-keeper down or credentials expired. Without PHPSESSID every check is skipped.
+        - alert: EduPhpsessidMissing
+          expr: |
+            edu_phpsessid_present == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "EDU_PHPSESSID missing"
+            description: "edu-master: EDU_PHPSESSID absent from redis for 10m. Webinar/diari/schedule checks are all skipped. Check session-keeper logs and EDU credentials."
+
+        # Hard deps: checker and playwright deployments unavailable.
+        - alert: WebinarCheckerDeploymentDown
+          expr: |
+            kube_deployment_status_replicas_unavailable{deployment="webinar-checker", namespace="edu-master"} > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker deployment unavailable"
+            description: "edu-master/webinar-checker deployment has {{ $value }} unavailable replica(s) for 10m."
+
+        - alert: PlaywrightServiceDown
+          expr: |
+            kube_deployment_status_replicas_unavailable{deployment="playwright-service", namespace="edu-master"} > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Playwright service unavailable"
+            description: "edu-master/playwright-service deployment has {{ $value }} unavailable replica(s) for 10m. All webinar/diari/schedule checks fail without it."

edu_master/k8s/playwright.yaml

@@ -17,7 +17,8 @@ spec:
     spec:
       containers:
         - name: playwright
-          image: mcr.microsoft.com/playwright:v1.63.0-jammy
+          # renovate: datasource=docker depName=mcr.microsoft.com/playwright versioning=docker
+          image: mcr.microsoft.com/playwright:v1.56.0-jammy
           imagePullPolicy: IfNotPresent
           command:
             - npx

edu_master/k8s/secrets.yaml.example

@@ -21,6 +21,8 @@ stringData:
   WEBINAR_TELEGRAM_TOKEN: ""
   WEBINAR_ADMIN_ID: ""
   WEBINAR_CHECK_INTERVAL: "60"
+  # Prometheus metrics endpoint (scraped via ServiceMonitor, alerts in k8s/alerts.yaml)
+  METRICS_PORT: "8000"
   # Database
   REDIS_HOST: "redis"
   REDIS_PORT: "6379"

edu_master/k8s/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: webinar-checker
+  namespace: edu-master
+  labels:
+    app: edu-master-webinar-checker
+spec:
+  selector:
+    app: edu-master-webinar-checker
+  ports:
+    - name: metrics
+      port: 8000
+      targetPort: metrics
+      protocol: TCP

edu_master/k8s/servicemonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: webinar-checker
+  namespace: edu-master
+  labels:
+    release: prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      app: edu-master-webinar-checker
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

edu_master/k8s/webinar-checker.yaml

@@ -47,6 +47,10 @@ spec:
         - name: webinar-checker
           image: gcr.forust.xyz/forust/webinar-checker:latest
           imagePullPolicy: Always
+          ports:
+            - name: metrics
+              containerPort: 8000
+              protocol: TCP
           envFrom:
             - secretRef:
                 name: edu-master-secrets

edu_master/webinar-checker/Dockerfile

@@ -2,8 +2,11 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Install dependencies
-RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==1.56.0 redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
+# renovate: datasource=pypi depName=playwright versioning=pep440
+ARG PLAYWRIGHT_VERSION=1.56.0
+
+# Install dependencies - PLAYWRIGHT_VERSION is single-source, renovate updates ARG above and all other places via regexManagers
+RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==${PLAYWRIGHT_VERSION} redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
 
 COPY checker.py .
 

edu_master/webinar-checker/checker.py

@@ -1,12 +1,15 @@
+import asyncio
 import contextlib
 import json
 import logging
 import os
 import re
 import tempfile
+import threading
 import time
 from datetime import datetime, timedelta
 from html import escape
+from http.server import BaseHTTPRequestHandler, HTTPServer
 
 import redis
 from playwright.async_api import async_playwright
@@ -48,6 +51,106 @@ USER_AGENT = _env(
 )
 WEBINAR_TELEGRAM_TOKEN = _env('WEBINAR_TELEGRAM_TOKEN')
 ADMIN_ID = int(_env('WEBINAR_ADMIN_ID', '0'))
+METRICS_PORT = int(_env('METRICS_PORT', '8000'))
+
+# --- Prometheus metrics (stdlib only, no extra deps) ---
+# Scraped by prometheus-stack via ServiceMonitor (edu_master/k8s/servicemonitor.yaml).
+# Critical alerts in edu_master/k8s/alerts.yaml fire to Telegram via Alertmanager.
+_METRICS_LOCK = threading.Lock()
+_METRICS = {
+    'last_run': 0.0,  # Unix ts of last check start
+    'last_success': 0.0,  # Unix ts of last successful check
+    'last_duration': 0.0,  # Duration of last check in seconds
+    'success_total': 0,
+    'failure_total': 0,
+    'consecutive_failures': 0,
+    'phpsessid_present': 1,  # 1 if EDU_PHPSESSID found in redis, else 0
+}
+
+
+def _metric_check_start():
+    with _METRICS_LOCK:
+        _METRICS['last_run'] = time.time()
+
+
+def _metric_check_ok(duration: float):
+    now = time.time()
+    with _METRICS_LOCK:
+        _METRICS['last_success'] = now
+        _METRICS['last_duration'] = duration
+        _METRICS['success_total'] += 1
+        _METRICS['consecutive_failures'] = 0
+        _METRICS['phpsessid_present'] = 1
+
+
+def _metric_check_fail(duration: float, phpsessid_missing: bool = False):
+    with _METRICS_LOCK:
+        _METRICS['last_duration'] = duration
+        _METRICS['failure_total'] += 1
+        _METRICS['consecutive_failures'] += 1
+        _METRICS['phpsessid_present'] = 0 if phpsessid_missing else 1
+
+
+def _metrics_render() -> bytes:
+    with _METRICS_LOCK:
+        m = dict(_METRICS)
+    lines = [
+        '# HELP webinar_check_last_run_timestamp_seconds Unix timestamp of last webinar check start.',
+        '# TYPE webinar_check_last_run_timestamp_seconds gauge',
+        f'webinar_check_last_run_timestamp_seconds {m["last_run"]}',
+        '# HELP webinar_check_last_success_timestamp_seconds Unix timestamp of last successful webinar check.',
+        '# TYPE webinar_check_last_success_timestamp_seconds gauge',
+        f'webinar_check_last_success_timestamp_seconds {m["last_success"]}',
+        '# HELP webinar_check_last_duration_seconds Duration of last webinar check in seconds.',
+        '# TYPE webinar_check_last_duration_seconds gauge',
+        f'webinar_check_last_duration_seconds {m["last_duration"]}',
+        '# HELP webinar_check_success_total Total successful webinar checks.',
+        '# TYPE webinar_check_success_total counter',
+        f'webinar_check_success_total {m["success_total"]}',
+        '# HELP webinar_check_failure_total Total failed webinar checks (timeout, playwright error, page error).',
+        '# TYPE webinar_check_failure_total counter',
+        f'webinar_check_failure_total {m["failure_total"]}',
+        '# HELP webinar_check_consecutive_failures Consecutive failed webinar checks (reset on success).',
+        '# TYPE webinar_check_consecutive_failures gauge',
+        f'webinar_check_consecutive_failures {m["consecutive_failures"]}',
+        '# HELP edu_phpsessid_present 1 if EDU_PHPSESSID exists in redis, 0 otherwise.',
+        '# TYPE edu_phpsessid_present gauge',
+        f'edu_phpsessid_present {m["phpsessid_present"]}',
+    ]
+    return ('\n'.join(lines) + '\n').encode()
+
+
+class _MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == '/metrics':
+            body = _metrics_render()
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain; version=0.0.4')
+            self.send_header('Content-Length', str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        elif self.path in ('/healthz', '/health'):
+            body = b'ok\n'
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain')
+            self.send_header('Content-Length', str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+    def log_message(self, *args):
+        pass  # keep bot logs clean
+
+
+def start_metrics_server(port: int = METRICS_PORT):
+    server = HTTPServer(('0.0.0.0', port), _MetricsHandler)  # noqa: S104 - k8s ServiceMonitor scrapes pod IP
+    thread = threading.Thread(target=server.serve_forever, name='metrics-server', daemon=True)
+    thread.start()
+    logger.info(f'Metrics server listening on :{port}/metrics')
+    return server
+
 
 # Redis Keys
 KEY_WHITELIST = 'bot:whitelist'
@@ -597,8 +700,9 @@ async def _collect_event_times(page) -> dict:
 async def fetch_diary_data(phpsessid: str) -> dict | None:
     logger.info('Fetching diary data via Playwright...')
     try:
+        async with asyncio.timeout(60):
             async with async_playwright() as p:
-            browser = await p.chromium.connect(PLAYWRIGHT_WS)
+                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
                 try:
                     context_browser = await browser.new_context(user_agent=USER_AGENT)
                     await context_browser.add_cookies(
@@ -607,7 +711,7 @@ async def fetch_diary_data(phpsessid: str) -> dict | None:
                     page = await context_browser.new_page()
 
                     try:
-                    await page.goto(DIARY_URL, wait_until='domcontentloaded')
+                        await asyncio.wait_for(page.goto(DIARY_URL, wait_until='domcontentloaded'), timeout=30)
                         await page.wait_for_selector('table.calendar', timeout=10000)
                         await page.wait_for_timeout(1500)
 
@@ -646,10 +750,16 @@ async def fetch_diary_data(phpsessid: str) -> dict | None:
                         logger.error(f'Error parsing diary: {e}')
                         return None
                     finally:
-                    await page.close()
-                    await context_browser.close()
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(page.close(), timeout=5)
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(context_browser.close(), timeout=5)
                 finally:
-                await browser.close()
+                    with contextlib.suppress(Exception):
+                        await asyncio.wait_for(browser.close(), timeout=5)
+    except TimeoutError:
+        logger.error('Diary fetch timed out (60s)')
+        return None
     except Exception as e:
         logger.error(f'Playwright error in diary fetch: {e}')
         return None
@@ -931,8 +1041,9 @@ def _parse_schedule_html(table_html: str) -> dict:
 async def fetch_schedule_data(phpsessid: str) -> dict | None:
     logger.info('Fetching schedule data via Playwright...')
     try:
+        async with asyncio.timeout(60):
             async with async_playwright() as p:
-            browser = await p.chromium.connect(PLAYWRIGHT_WS)
+                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
                 try:
                     context_browser = await browser.new_context(user_agent=USER_AGENT)
                     await context_browser.add_cookies(
@@ -941,7 +1052,7 @@ async def fetch_schedule_data(phpsessid: str) -> dict | None:
                     page = await context_browser.new_page()
 
                     try:
-                    await page.goto(SCHEDULE_URL, wait_until='domcontentloaded')
+                        await asyncio.wait_for(page.goto(SCHEDULE_URL, wait_until='domcontentloaded'), timeout=30)
                         await page.wait_for_selector('table.schedule-table', timeout=10000)
                         await page.wait_for_timeout(1500)
 
@@ -969,10 +1080,16 @@ async def fetch_schedule_data(phpsessid: str) -> dict | None:
                         logger.error(f'Error parsing schedule: {e}')
                         return None
                     finally:
-                    await page.close()
-                    await context_browser.close()
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(page.close(), timeout=5)
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(context_browser.close(), timeout=5)
                 finally:
-                await browser.close()
+                    with contextlib.suppress(Exception):
+                        await asyncio.wait_for(browser.close(), timeout=5)
+    except TimeoutError:
+        logger.error('Schedule fetch timed out (60s)')
+        return None
     except Exception as e:
         logger.error(f'Playwright error in schedule fetch: {e}')
         return None
@@ -1485,10 +1602,13 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
         int: Number of webinars found, or None if check failed
     """
     logger.info('Running webinar check...')
+    _t0 = time.time()
+    _metric_check_start()
 
     phpsessid = redis_client.get(KEY_PHPSESSID)
     if not phpsessid:
         logger.warning('PHPSESSID missing. Skipping check.')
+        _metric_check_fail(time.time() - _t0, phpsessid_missing=True)
         # --- DEBUG LOGGING ---
         try:
             with open('phpsessid_missing.log', 'a') as f:
@@ -1502,9 +1622,10 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
     content = ''
 
     try:
+        async with asyncio.timeout(90):
             async with async_playwright() as p:
                 # Connect to remote Playwright service
-            browser = await p.chromium.connect(PLAYWRIGHT_WS)
+                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
 
                 try:
                     # Create browser context with user agent
@@ -1520,7 +1641,7 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
 
                     try:
                         # Navigate to webinar page
-                    await page.goto(WEBINAR_URL, wait_until='domcontentloaded')
+                        await asyncio.wait_for(page.goto(WEBINAR_URL, wait_until='domcontentloaded'), timeout=30)
 
                         # Wait for the table to load
                         await page.wait_for_selector('#meetings table', timeout=10000)
@@ -1564,16 +1685,25 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
                             if page and not content:
                                 content = await page.content()
 
+                        _metric_check_fail(time.time() - _t0)
                         return None
                     finally:
-                    await page.close()
-                    await context_browser.close()
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(page.close(), timeout=5)
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(context_browser.close(), timeout=5)
 
                 finally:
-                await browser.close()
+                    with contextlib.suppress(Exception):
+                        await asyncio.wait_for(browser.close(), timeout=5)
 
+    except TimeoutError:
+        logger.error('Webinar check timed out after 90s (playwright hang)')
+        _metric_check_fail(time.time() - _t0)
+        return None
     except Exception as e:
         logger.error(f'Playwright error: {e}')
+        _metric_check_fail(time.time() - _t0)
         return None
 
     # --- DEBUG LOGGING (Saving last response content) ---
@@ -1637,6 +1767,7 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
         else:
             logger.info(f'Found {len(current_webinars)} webinar(s), but all are already known')
 
+    _metric_check_ok(time.time() - _t0)
     return len(current_webinars)
 
 
@@ -1680,6 +1811,8 @@ def main():
     job_queue = app.job_queue
     job_queue.run_repeating(check_webinars_job, interval=WEBINAR_CHECK_INTERVAL, first=10)
 
+    start_metrics_server()
+
     logger.info('Bot started polling...')
     app.run_polling()
 

gitea/k8s/certificates.yaml

@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: gitea-prod-tls
+  namespace: gitea
+spec:
+  secretName: gitea-prod-tls
+  dnsNames:
+    - gcr.forust.xyz
+    - gitea.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: gitea
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

gitea/k8s/gitea.yaml

@@ -31,7 +31,7 @@ spec:
     spec:
       containers:
         - name: gitea
-          image: docker.gitea.com/gitea:1.27.3
+          image: gitea/gitea:1.27.3
           envFrom:
             - configMapRef:
                 name: gitea-config

gitea/k8s/ingress.yaml

@@ -24,7 +24,7 @@ spec:
         - name: gitea-service
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: gitea-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -45,6 +45,9 @@ spec:
       services:
         - name: gitea-service
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRouteTCP

glance/k8s/certificates.yaml

@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: glance-prod-tls
+  namespace: glance
+spec:
+  secretName: glance-prod-tls
+  dnsNames:
+    - forust.xyz
+    - www.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: glance
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

glance/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: glance-service
           port: 8080
   tls:
-    certResolver: letsencrypt
+    secretName: glance-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -35,6 +35,9 @@ spec:
       services:
         - name: glance-service
           port: 8080
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: Middleware

headscale/k8s/routing/certificates.yaml

@@ -0,0 +1,41 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: headscale-prod-tls
+  namespace: headscale
+spec:
+  secretName: headscale-prod-tls
+  dnsNames:
+    - hs.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: headplane-prod-tls
+  namespace: headscale
+spec:
+  secretName: headplane-prod-tls
+  dnsNames:
+    - hp.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: headscale
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

headscale/k8s/routing/ingress.yaml

@@ -38,7 +38,7 @@ spec:
         - name: headscale-server-external
           port: 9090
   tls:
-    certResolver: letsencrypt
+    secretName: headscale-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -64,7 +64,7 @@ spec:
         - name: headplane-external
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: headplane-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -90,6 +90,9 @@ spec:
       services:
         - name: headscale-server-external
           port: 9090
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -110,3 +113,5 @@ spec:
       services:
         - name: headplane-external
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls

homepages/k8s/certificates.yaml

@@ -0,0 +1,42 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: forust-homepage-prod-tls
+  namespace: homepages
+spec:
+  secretName: forust-homepage-prod-tls
+  dnsNames:
+    - forust.xyz
+    - www.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: xdfnx-homepage-prod-tls
+  namespace: homepages
+spec:
+  secretName: xdfnx-homepage-prod-tls
+  dnsNames:
+    - xdfnx.cfd
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: homepages
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

homepages/k8s/ingress.yaml

@@ -17,7 +17,7 @@ spec:
         - name: forust-homepage-service
           port: 80
   tls:
-    certResolver: letsencrypt
+    secretName: forust-homepage-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -34,6 +34,9 @@ spec:
       services:
         - name: forust-homepage-service
           port: 80
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -53,7 +56,7 @@ spec:
         - name: xdfnx-homepage-service
           port: 80
   tls:
-    certResolver: letsencrypt
+    secretName: xdfnx-homepage-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -69,3 +72,5 @@ spec:
       services:
         - name: xdfnx-homepage-service
           port: 80
+  tls:
+    secretName: internal-wildcard-tls

kener/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: kener-service
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: kener-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -32,3 +32,5 @@ spec:
       services:
         - name: kener-service
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls

loki/k8s/alloy-values.yaml

@@ -0,0 +1,57 @@
+# Pinned chart: grafana/alloy 1.12.1 (app v1.19.2).
+# Install (deferred to deploy task, namespace prometheus):
+#   helm upgrade --install alloy grafana/alloy --version 1.12.1 \
+#     --namespace prometheus --values loki/k8s/alloy-values.yaml --wait
+# DaemonSet ships k8s pod logs (API-tailed, no hostPath mounts) to Loki.
+# Scope phase 1: k8s only, compose leftovers out.
+
+controller:
+  type: daemonset
+  resources:
+    requests:
+      memory: "128Mi"
+      cpu: "50m"
+    limits:
+      memory: "512Mi"
+      cpu: "500m"
+
+image:
+  tag: "v1.19.2"
+
+alloy:
+  configMap:
+    create: true
+    content: |
+      discovery.kubernetes "pods" {
+        role = "pod"
+      }
+
+      discovery.relabel "pods" {
+        targets = discovery.kubernetes.pods.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace"]
+          target_label  = "namespace"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_name"]
+          target_label  = "pod"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_name"]
+          target_label  = "container"
+        }
+      }
+
+      loki.source.kubernetes "pods" {
+        targets    = discovery.relabel.pods.output
+        forward_to = [loki.write.default.receiver]
+      }
+
+      loki.write "default" {
+        endpoint {
+          url = "http://loki-gateway.prometheus.svc.cluster.local/loki/api/v1/push"
+        }
+      }

loki/k8s/loki-values.yaml

@@ -0,0 +1,97 @@
+# Pinned chart: grafana/loki 7.3.0 (app 3.6.12).
+# Install (deferred to deploy task, namespace prometheus):
+#   helm upgrade --install loki grafana/loki --version 7.3.0 \
+#     --namespace prometheus --values loki/k8s/loki-values.yaml --wait
+# SingleBinary, filesystem storage on local-path-retain, 14d retention.
+# No IngressRoute: Loki is cluster-internal, queried via Grafana datasource.
+
+deploymentMode: SingleBinary
+
+loki:
+  # Multitenancy off: single-node homelab, gateway + Alloy + Grafana talk to one tenant.
+  auth_enabled: false
+  image:
+    tag: "3.6.12"
+  commonConfig:
+    # Single replica: default RF=3 would require 3 ingesters and fail all writes.
+    replication_factor: 1
+  storage:
+    type: filesystem
+  schemaConfig:
+    configs:
+      - from: "2024-04-01"
+        store: tsdb
+        object_store: filesystem
+        schema: v13
+        index:
+          prefix: index_
+          period: 24h
+  compactor:
+    retention_enabled: true
+    delete_request_store: filesystem
+  limits_config:
+    retention_period: 336h
+  rulerConfig:
+    wal:
+      dir: /var/loki/ruler-wal
+    storage:
+      type: local
+      local:
+        directory: /var/loki/rules
+
+singleBinary:
+  replicas: 1
+  persistence:
+    enabled: true
+    size: 20Gi
+    storageClass: local-path-retain
+  resources:
+    requests:
+      memory: "512Mi"
+      cpu: "200m"
+    limits:
+      memory: "2Gi"
+      cpu: "1000m"
+
+# Zeroed: unused in SingleBinary mode (chart validation requires it).
+write:
+  replicas: 0
+read:
+  replicas: 0
+backend:
+  replicas: 0
+
+gateway:
+  replicas: 1
+  resources:
+    requests:
+      memory: "64Mi"
+      cpu: "50m"
+    limits:
+      memory: "256Mi"
+      cpu: "300m"
+
+monitoring:
+  serviceMonitor:
+    enabled: true
+    labels:
+      release: prometheus-stack
+    interval: 15s
+  rules:
+    enabled: true
+    namespace: prometheus
+    labels:
+      release: prometheus-stack
+
+# Disabled: memcached caches don't fit a memory-tight single node.
+# SingleBinary works without them (slower repeated queries, fine at homelab scale).
+resultsCache:
+  enabled: false
+chunksCache:
+  enabled: false
+
+# Disabled: synthetic canary traffic + helm test pod, noise on a single node.
+lokiCanary:
+  enabled: false
+test:
+  enabled: false

metube/compose.yaml

@@ -1,6 +1,6 @@
 services:
   metube:
-    image: ghcr.io/alexta69/metube:2026.08.28
+    image: ghcr.io/alexta69/metube:2026.09.15
     container_name: metube
     restart: unless-stopped
     # ports:

metube/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: metube-prod-tls
+  namespace: metube
+spec:
+  secretName: metube-prod-tls
+  dnsNames:
+    - metube.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: metube
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

metube/k8s/ingress.yaml

@@ -15,7 +15,7 @@ spec:
         - name: metube-service
           port: 8081
   tls:
-    certResolver: letsencrypt
+    secretName: metube-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,3 +31,5 @@ spec:
       services:
         - name: metube-service
           port: 8081
+  tls:
+    secretName: internal-wildcard-tls

metube/k8s/metube.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: metube
-          image: ghcr.io/alexta69/metube:2026.08.28
+          image: ghcr.io/alexta69/metube:2026.09.15
           envFrom:
             - configMapRef:
                 name: metube-config

n8n/compose.yaml

@@ -1,6 +1,6 @@
 services:
   n8n:
-    image: docker.n8n.io/n8nio/n8n:2.39.5
+    image: docker.n8n.io/n8nio/n8n:2.40.3
     container_name: n8n
     restart: unless-stopped
     environment:

n8n/k8s/ingress.yaml

@@ -32,3 +32,5 @@ spec:
       services:
         - name: n8n-service
           port: 5678
+  tls:
+    secretName: internal-wildcard-tls

n8n/k8s/internal-certificate.yaml

@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: n8n
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

n8n/k8s/n8n.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: n8n
-          image: docker.n8n.io/n8nio/n8n:2.39.5
+          image: docker.n8n.io/n8nio/n8n:2.40.3
           envFrom:
             - configMapRef:
                 name: n8n-config

netronome/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: netronome-prod-tls
+  namespace: netronome
+spec:
+  secretName: netronome-prod-tls
+  dnsNames:
+    - nm.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: netronome
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

netronome/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: netronome-service
           port: 7575
   tls:
-    certResolver: letsencrypt
+    secretName: netronome-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -32,3 +32,5 @@ spec:
       services:
         - name: netronome-service
           port: 7575
+  tls:
+    secretName: internal-wildcard-tls

nextcloud/k8s/routing/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: nextcloud-prod-tls
+  namespace: nextcloud
+spec:
+  secretName: nextcloud-prod-tls
+  dnsNames:
+    - nextcloud.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: nextcloud
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

nextcloud/k8s/routing/ingress.yaml

@@ -18,7 +18,7 @@ spec:
         - name: nextcloud-apache
           port: 11000
   tls:
-    certResolver: letsencrypt
+    secretName: nextcloud-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -58,6 +58,9 @@ spec:
   #         serversTransport: insecure-transport
   # tls:
   #   certResolver: letsencrypt
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -75,3 +78,5 @@ spec:
           port: 8888
           scheme: https
           serversTransport: insecure-transport
+  tls:
+    secretName: internal-wildcard-tls

portainer/compose.yaml

@@ -1,6 +1,6 @@
 services:
   portainer:
-    image: portainer/portainer-ce:2.45.0
+    image: portainer/portainer-ce:2.45.1
     container_name: portainer
     restart: always
     volumes:

portainer/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: portainer-prod-tls
+  namespace: portainer
+spec:
+  secretName: portainer-prod-tls
+  dnsNames:
+    - portainer.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: portainer
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

portainer/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: portainer-service
           port: 9000
   tls:
-    certResolver: letsencrypt
+    secretName: portainer-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -32,3 +32,5 @@ spec:
       services:
         - name: portainer-service
           port: 9000
+  tls:
+    secretName: internal-wildcard-tls

portainer/k8s/portainer.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: portainer
-          image: portainer/portainer-ce:2.45.0
+          image: portainer/portainer-ce:2.45.1
           ports:
             - containerPort: 9000
           volumeMounts:

postgres/.env.example

@@ -3,3 +3,4 @@ AUTHENTIK_DB_PASSWORD=
 GITEA_DB_PASSWORD=
 NETRONOME_DB_PASSWORD=
 PENPOT_DB_PASSWORD=
+STATUSPAGE_DB_PASSWORD=

postgres/README.md

@@ -1,23 +1,22 @@
 # Shared PostgreSQL
 
-This directory contains a PostgreSQL 15 deployment draft for Authentik, Gitea,
-Netronome, and Penpot. It creates one database and one login role per service;
-it does not migrate existing data or change application connection settings.
+This directory contains the shared PostgreSQL 17 deployment for Authentik,
+Gitea, Netronome, and Statuspage. It creates one database and one login role
+per service. Per-service standalone databases were removed after the
+migration (Sep 2026); Penpot stays on its own compose PostgreSQL (archived,
+not part of the shared instance).
 
 ## Compatibility baseline
 
-| Service   | Current application | Current standalone PostgreSQL | Common PostgreSQL 15                                                                 |
-| --------- | ------------------- | ----------------------------: | ------------------------------------------------------------------------------------ |
-| Authentik | 2025.10.2           |                            15 | Supported (Authentik requires 14+)                                                   |
-| Gitea     | 1.27.3              |                            14 | Supported (Gitea requires 12+)                                                       |
-| Netronome | 0.14.0              |                            17 | Validate in staging; upstream's example uses 17 but no 17-only feature is documented |
-| Penpot    | 2.17.2              |                            15 | Supported by the official deployment                                                 |
+| Service    | Current application | Shared PostgreSQL 17                   |
+| ---------- | ------------------- | -------------------------------------- |
+| Authentik  | 2025.10.x           | Supported (Authentik requires 14+)     |
+| Gitea      | 1.27.3              | Supported (Gitea requires 12+)         |
+| Netronome  | 0.14.0              | Supported (upstream's example uses 17) |
+| Statuspage | custom              | Supported                              |
 
-PostgreSQL 15 is the conservative common major. A major-version downgrade or
-change must use a logical dump/restore; changing only the image tag while
-keeping a data directory is not supported. Back up and migrate one application
-at a time, starting with Netronome because its current standalone deployment
-uses PostgreSQL 17.
+A major-version change must use a logical dump/restore; changing only the
+image tag while keeping a data directory is not supported.
 
 For Compose, copy `.env.example` to `.env`, set all passwords, and start it with
 `docker compose -f shared-compose.yaml up -d`. This file is intentionally not

postgres/initdb/01-create-databases.sh

@@ -5,12 +5,12 @@ set -euo pipefail
 : "${GITEA_DB_PASSWORD:?GITEA_DB_PASSWORD is required}"
 : "${NETRONOME_DB_PASSWORD:?NETRONOME_DB_PASSWORD is required}"
 : "${PENPOT_DB_PASSWORD:?PENPOT_DB_PASSWORD is required}"
+: "${STATUSPAGE_DB_PASSWORD:?STATUSPAGE_DB_PASSWORD is required}"
 
 create_role_and_database() {
   local role="$1"
   local database="$2"
   local password="$3"
-
   psql --username "$POSTGRES_USER" --dbname postgres \
     -v role="$role" -v database="$database" -v password="$password" \
     <<'SQL'
@@ -25,3 +25,4 @@ create_role_and_database authentik authentik "$AUTHENTIK_DB_PASSWORD"
 create_role_and_database gitea gitea "$GITEA_DB_PASSWORD"
 create_role_and_database netronome netronome "$NETRONOME_DB_PASSWORD"
 create_role_and_database penpot penpot "$PENPOT_DB_PASSWORD"
+create_role_and_database statuspage statuspage "$STATUSPAGE_DB_PASSWORD"

postgres/k8s/postgres.yaml

@@ -32,7 +32,7 @@ spec:
     spec:
       containers:
         - name: postgres
-          image: postgres:17.6-alpine
+          image: postgres:17.11-alpine
           ports:
             - name: postgres
               containerPort: 5432
@@ -60,11 +60,23 @@ spec:
               command: ["pg_isready", "-U", "postgres", "-d", "postgres"]
             initialDelaySeconds: 10
             periodSeconds: 10
+          startupProbe:
+            exec:
+              command: ["pg_isready", "-U", "postgres", "-d", "postgres"]
+            failureThreshold: 30
+            periodSeconds: 10
           livenessProbe:
             exec:
               command: ["pg_isready", "-U", "postgres", "-d", "postgres"]
             initialDelaySeconds: 30
             periodSeconds: 20
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "500m"
+            limits:
+              memory: "2Gi"
+              cpu: "2000m"
       volumes:
         - name: postgres-data
           persistentVolumeClaim:

postgres/shared-compose.yaml

@@ -1,6 +1,6 @@
 services:
   postgres:
-    image: postgres:15.19-alpine
+    image: postgres:17.11-alpine
     container_name: homelab-postgres
     restart: unless-stopped
     env_file:

prometheus-stack/alertmanager.yaml

@@ -1,5 +1,10 @@
 route:
   receiver: default
+  routes:
+    - receiver: "null"
+      matchers:
+        - alertname="Watchdog"
 
 receivers:
   - name: default
+  - name: "null"

prometheus-stack/compose.yaml

@@ -1,6 +1,6 @@
 services:
   grafana:
-    image: grafana/grafana:13.2.1
+    image: grafana/grafana:13.2.2
     container_name: prometheus-grafana
     restart: unless-stopped
     env_file:
@@ -44,7 +44,7 @@ services:
       - proxy
 
   alertmanager:
-    image: prom/alertmanager:v0.34.0
+    image: prom/alertmanager:v0.34.1
     container_name: prometheus-alertmanager
     restart: unless-stopped
     command:

prometheus-stack/k8s/alertmanager-config.example.yaml

@@ -1,3 +1,6 @@
+templates:
+  - "/etc/alertmanager/config/telegram.tmpl"
+
 route:
   receiver: telegram
   group_by:
@@ -7,10 +10,10 @@ route:
   group_interval: 5m
   repeat_interval: 12h
   routes:
-    - receiver: null
+    - receiver: "null"
       matchers:
         - alertname="InfoInhibitor"
-    - receiver: null
+    - receiver: "null"
       matchers:
         - alertname="Watchdog"
 
@@ -35,4 +38,5 @@ receivers:
         chat_id: REPLACE_WITH_TELEGRAM_CHAT_ID
         parse_mode: HTML
         send_resolved: true
-  - name: null
+        message: '{{ template "telegram.forust.message" . }}'
+  - name: "null"

prometheus-stack/k8s/alertmanager-telegram.tmpl

@@ -0,0 +1,44 @@
+{{- define "telegram.forust.message" -}}
+{{- $statusEmoji := "🚨" -}}
+{{- if ne .Status "firing" -}}{{- $statusEmoji = "✅" -}}{{- end -}}
+{{- $count := len .Alerts.Firing -}}
+{{- if eq .Status "resolved" -}}{{- $count = len .Alerts.Resolved -}}{{- end -}}
+{{ $statusEmoji }} <b>{{ .Status | toUpper }} ({{ $count }})</b>
+{{- range .Alerts }}
+
+<b>{{ .Labels.alertname }}</b>
+{{- $sev := .Labels.severity }}
+{{- if eq $sev "critical" }} 🔥 critical
+{{- else if eq $sev "warning" }} ⚠️ warning
+{{- else if eq $sev "info" }} ℹ️ info
+{{- else if $sev }} • {{ $sev }}
+{{- end }}
+{{- if .Annotations.description }}
+
+<i>{{ .Annotations.description }}</i>
+{{- end }}
+{{- if .Labels.namespace }}
+📦 Namespace: <code>{{ .Labels.namespace }}</code>
+{{- end }}
+{{- if .Labels.pod }}
+📦 Pod: <code>{{ .Labels.pod }}</code>
+{{- end }}
+{{- if .Labels.container }}
+🐳 Container: <code>{{ .Labels.container }}</code>
+{{- end }}
+{{- if .Labels.node }}
+🖥 Node: <code>{{ .Labels.node }}</code>
+{{- end }}
+{{- if .Labels.instance }}
+🖥 Instance: <code>{{ .Labels.instance }}</code>
+{{- end }}
+{{- if .Labels.job }}
+🔧 Job: <code>{{ .Labels.job }}</code>
+{{- end }}
+{{- if eq .Status "firing" }}
+🕐 Since: <code>{{ .StartsAt | date "2006-01-02 15:04 MST" }}</code>
+{{- else }}
+🕐 Resolved: <code>{{ .EndsAt | date "2006-01-02 15:04 MST" }}</code>
+{{- end }}
+{{- end }}
+{{- end }}

prometheus-stack/k8s/alerts.yaml

@@ -66,3 +66,21 @@ spec:
           annotations:
             summary: "Node memory pressure"
             description: "Node {{ $labels.instance }} has used more than 90% of memory for 15 minutes."
+
+        - alert: LokiDown
+          expr: kube_statefulset_status_replicas_unavailable{statefulset="loki"} > 0 or kube_deployment_status_replicas_unavailable{deployment="loki-gateway"} > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Loki is down"
+            description: "Loki in namespace {{ $labels.namespace }} has unavailable replicas for more than 10 minutes. Logs are not queryable."
+
+        - alert: AlloyDown
+          expr: kube_daemonset_status_number_unavailable{daemonset="alloy"} > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Alloy is down"
+            description: "Alloy DaemonSet in namespace {{ $labels.namespace }} has {{ $value }} unavailable pods for more than 10 minutes. Pod logs are not being shipped to Loki."

prometheus-stack/k8s/cert-manager-alerts.yaml

@@ -0,0 +1,43 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cert-manager
+  namespace: prometheus
+  labels:
+    release: prometheus-stack
+spec:
+  groups:
+    - name: cert-manager
+      rules:
+        - alert: CertManagerCertNotReady
+          expr: certmanager_certificate_ready_status{condition="False"} == 1
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Certificate {{ $labels.exported_namespace }}/{{ $labels.name }} is not ready"
+            description: "Certificate has been failing for more than 10 minutes. Check Challenges/Orders in that namespace."
+        - alert: CertManagerCertExpirySoon
+          expr: certmanager_certificate_expiration_timestamp_seconds - time() < 86400 * 14
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            summary: "Certificate {{ $labels.exported_namespace }}/{{ $labels.name }} expires in less than 14 days"
+            description: "Renewal should have happened automatically. Check cert-manager logs if it persists."
+        - alert: CertManagerCertExpiryCritical
+          expr: certmanager_certificate_expiration_timestamp_seconds - time() < 86400 * 7
+          for: 1h
+          labels:
+            severity: critical
+          annotations:
+            summary: "Certificate {{ $labels.exported_namespace }}/{{ $labels.name }} expires in less than 7 days"
+            description: "Manual intervention likely needed: cmctl status certificate {{ $labels.name }} -n {{ $labels.exported_namespace }}."
+        - alert: CertManagerHittingRateLimits
+          expr: sum by (exported_namespace) (rate(certmanager_http_acme_client_request_count{status=~"429.*"}[10m])) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "cert-manager is hitting ACME rate limits"
+            description: "Back off manual issuance retries and check failed Orders/Challenges."

prometheus-stack/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: grafana-prod-tls
+  namespace: prometheus
+spec:
+  secretName: grafana-prod-tls
+  dnsNames:
+    - grafana.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: prometheus
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

prometheus-stack/k8s/grafana-values.yaml

@@ -14,8 +14,11 @@ grafana:
 
   persistence:
     enabled: true
-    storageClassName: local-path-retain
-    size: 20Gi
+    # Matches live PVC (10Gi/local-path). Retain migration is a separate task
+    # with data migration (see storage-audit doc); do NOT change SC/size here
+    # without migrating, helm upgrade fails on immutable PVC fields.
+    storageClassName: local-path
+    size: 10Gi
 
   ingress:
     enabled: false
@@ -23,6 +26,12 @@ grafana:
   service:
     port: 80
 
+  additionalDataSources:
+    - name: Loki
+      type: loki
+      url: http://loki-gateway.prometheus.svc.cluster.local
+      access: proxy
+
 prometheus:
   prometheusSpec:
     retention: 60d
@@ -30,7 +39,8 @@ prometheus:
     storageSpec:
       volumeClaimTemplate:
         spec:
-          storageClassName: "local-path-retain"
+          # Matches live PVC, see note on grafana.persistence above.
+          storageClassName: "local-path"
           accessModes:
             - ReadWriteOnce
           resources:
@@ -48,12 +58,13 @@ alertmanager:
     storage:
       volumeClaimTemplate:
         spec:
-          storageClassName: local-path-retain
+          # Matches live PVC (20Gi), see note on grafana.persistence above.
+          storageClassName: local-path
           accessModes:
             - ReadWriteOnce
           resources:
             requests:
-              storage: 10Gi
+              storage: 20Gi
 
 defaultRules:
   disabled:

prometheus-stack/k8s/ingress.yaml

@@ -12,14 +12,11 @@ spec:
       middlewares:
         - name: crowdsec-bouncer
           namespace: crowdsec
-        - name: "security-chain@file"
       services:
         - name: prometheus-stack-grafana
           port: 80
   tls:
-    certResolver: letsencrypt
-    domains:
-      - main: grafana.forust.xyz
+    secretName: grafana-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -35,3 +32,5 @@ spec:
       services:
         - name: prometheus-stack-grafana
           port: 80
+  tls:
+    secretName: internal-wildcard-tls

prometheus-stack/k8s/traefik-alerts.yaml

@@ -42,7 +42,7 @@ spec:
         - alert: TraefikServiceHighLatency
           expr: |
             histogram_quantile(0.95,
-              sum(rate(traefik_service_request_duration_seconds_bucket[5m])) by (le, service)) > 2
+              sum(rate(traefik_service_request_duration_seconds_bucket{service!~"xui-xui-service-.*"}[5m])) by (le, service)) > 2
           for: 5m
           labels:
             severity: warning

reloader/k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: reloader

renovate.json

@@ -1,11 +1,43 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": ["config:recommended"],
-  "enabledManagers": ["docker-compose", "kubernetes"],
+  "enabledManagers": ["dockerfile", "docker-compose", "kubernetes", "helm-values", "custom.regex"],
+  "helm-values": {
+    "managerFilePatterns": ["/k8s/.+values\\.ya?ml$/"]
+  },
   "kubernetes": {
     "managerFilePatterns": ["/k8s/.+\\.ya?ml$/"]
   },
+  "customManagers": [
+    {
+      "customType": "regex",
+      "description": "singlesource: playwright npm version pinned in npx command (k8s + compose)",
+      "fileMatch": ["^edu_master/k8s/playwright\\.yaml$", "^edu_master/compose\\.yaml$"],
+      "matchStrings": ["playwright@(?<currentValue>\\d+\\.\\d+\\.\\d+)"],
+      "datasourceTemplate": "npm",
+      "depNameTemplate": "playwright"
+    },
+    {
+      "customType": "regex",
+      "description": "singlesource: PLAYWRIGHT_VERSION file",
+      "fileMatch": ["^edu_master/PLAYWRIGHT_VERSION$"],
+      "matchStrings": ["^(?<currentValue>\\d+\\.\\d+\\.\\d+)$"],
+      "datasourceTemplate": "pypi",
+      "depNameTemplate": "playwright"
+    }
+  ],
   "packageRules": [
+    {
+      "description": "singlesource playwright - use whichever version is found, keep docker+pypi+npm in sync",
+      "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
+      "groupName": "playwright singlesource",
+      "groupSlug": "playwright"
+    },
+    {
+      "description": "playwright must not automerge - version skew breaks WS handshake (checker.py:1523 vs playwright.yaml:20)",
+      "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
+      "automerge": false
+    },
     {
       "description": "Keep private homelab images unchanged",
       "matchDatasources": ["docker"],

renovate/README.md

@@ -24,12 +24,25 @@ The `renovate/k8s/active` marker makes the normal deployment workflow include
 the namespace, ConfigMap, and CronJob. The Secret is intentionally excluded
 from Git and must be applied separately after every new cluster.
 
-Run it immediately instead of waiting for the six-hour schedule:
+Run it immediately instead of waiting for the six-hour schedule.
+
+Two options, both use the same `renovate/config.js`:
 
 ```sh
 kubectl create job --from=cronjob/renovate renovate-manual-$(date +%s) -n renovate
 ```
 
+or the `renovate-run` Actions workflow (Actions tab → `renovate-run` →
+Run workflow). It runs `renovate/renovate:44.103.0` on the self-hosted
+runner via Docker. Required Actions secrets (repo or org settings):
+
+- `RENOVATE_TOKEN` — renovate-bot PAT (repository + issue read/write).
+- `RENOVATE_GITHUB_COM_TOKEN` — optional, for changelogs and GitHub rate limits.
+
+Inputs: `repositories` (default `forust/homelab`), `log_level`
+(`info`/`debug`). Only one run at a time (concurrency group
+`renovate-run`), same as the CronJob `Forbid` policy.
+
 Inspect runs with:
 
 ```sh

renovate/config.js

@@ -1,7 +1,10 @@
 module.exports = {
   platform: 'gitea',
-  endpoint: process.env.RENOVATE_ENDPOINT,
-  enabledManagers: ['docker-compose', 'kubernetes'],
+  endpoint: process.env.RENOVATE_ENDPOINT || 'https://gitea.forust.xyz/api/v1',
+  enabledManagers: ['docker-compose', 'kubernetes', 'helm-values'],
+  'helm-values': {
+    managerFilePatterns: ['/k8s/.+values\\.ya?ml$/'],
+  },
   kubernetes: {
     managerFilePatterns: ['/k8s/.+\\.ya?ml$/'],
   },

renovate/k8s/configmap.yaml

@@ -7,8 +7,11 @@ data:
   config.js: |
     module.exports = {
       platform: 'gitea',
-      endpoint: process.env.RENOVATE_ENDPOINT,
-      enabledManagers: ['docker-compose', 'kubernetes'],
+      endpoint: process.env.RENOVATE_ENDPOINT || 'https://gitea.forust.xyz/api/v1',
+      enabledManagers: ['docker-compose', 'kubernetes', 'helm-values'],
+      'helm-values': {
+        managerFilePatterns: ['/k8s/.+values\\.ya?ml$/'],
+      },
       kubernetes: {
         managerFilePatterns: ['/k8s/.+\\.ya?ml$/'],
       },
@@ -22,7 +25,10 @@ data:
       dependencyDashboard: true,
       prCreation: 'immediate',
       labels: ['dependencies', 'automated'],
-      extends: ['config:recommended', ':dependencyDashboard'],
+      extends: [
+        'config:recommended',
+        ':dependencyDashboard',
+      ],
       packageRules: [
         {
           description: 'Do not update private homelab images',

renovate/k8s/cronjob.yaml

@@ -16,7 +16,7 @@ spec:
           restartPolicy: Never
           containers:
             - name: renovate
-              image: renovate/renovate:44.85.0
+              image: renovate/renovate:44.106.0
               env:
                 - name: RENOVATE_PLATFORM
                   value: gitea

renovate/renovate-compose.yaml

@@ -1,6 +1,6 @@
 services:
   renovate:
-    image: renovate/renovate:44.83.2
+    image: renovate/renovate:44.103.0
     container_name: renovate
     restart: "no"
     env_file:

searxng/k8s/certificates.yaml

@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: searxng-prod-tls
+  namespace: searxng
+spec:
+  secretName: searxng-prod-tls
+  dnsNames:
+    - s.forust.xyz
+    - search.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: searxng
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

searxng/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: searxng-service
           port: 8080
   tls:
-    certResolver: letsencrypt
+    secretName: searxng-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -32,3 +32,5 @@ spec:
       services:
         - name: searxng-service
           port: 8080
+  tls:
+    secretName: internal-wildcard-tls

termix/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: termix-prod-tls
+  namespace: termix
+spec:
+  secretName: termix-prod-tls
+  dnsNames:
+    - termix.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: termix
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer