chore(deps): update container patch updates

Reviewed-on: #59

.gitea/actionlint.yaml

@@ -0,0 +1,10 @@
+# actionlint configuration. Passed explicitly from the ci workflow:
+#   actionlint -config-file .gitea/actionlint.yaml .gitea/workflows/*.yaml
+#
+# The self-hosted act_runner registers custom labels that actionlint cannot know
+# about, so declare them here instead of silencing the whole runner-label check.
+self-hosted-runner:
+  labels:
+    - arch
+    - homelab
+    - prod

.gitea/workflows/ci.yaml

@@ -7,6 +7,12 @@ on:
   pull_request:
   workflow_dispatch:
 
+# Every job here is checkout plus local tools. The token needs to read the tree
+# and nothing else, and saying so keeps a future step that reaches for the API
+# from quietly holding a token that can write to the repository.
+permissions:
+  contents: read
+
 concurrency:
   group: ci-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
@@ -15,8 +21,90 @@ env:
   REGISTRY: gcr.forust.xyz
 
 jobs:
+  lint-compose:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # Structure check for every committed Compose file, active or not.
+      # Interpolation, env-file and bind-mount resolution are all switched off,
+      # because inactive stacks have no .env here and would only fail on their
+      # ${VAR:?} guards. Active stacks get the full check with interpolation in
+      # the deploy workflow, where the real .env files live.
+      - name: Validate Compose files
+        shell: bash
+        run: |
+          set -euo pipefail
+          source .gitea/workflows/compose-lint.sh
+
+          mapfile -t safe_flags < <(compose_safe_flags)
+          echo "docker compose config ${safe_flags[*]-}"
+
+          mapfile -t files < <(compose_files)
+          if [ "${#files[@]}" -eq 0 ]; then
+            echo "No Compose files found."
+            exit 0
+          fi
+
+          failed=0
+          for f in "${files[@]}"; do
+            if ! out="$(validate_compose_file "$f" ${safe_flags[@]+"${safe_flags[@]}"} 2>&1)"; then
+              failed=1
+              echo "::error file=${f}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+
+          if [ "$failed" -ne 0 ]; then
+            echo "Compose validation failed."
+            exit 1
+          fi
+          echo "checked ${#files[@]} Compose file(s)"
+
+  lint-actionlint:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Lint Gitea Actions workflows with actionlint
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh actionlint)"
+          export PATH="$tools_dir:$PATH"
+          actionlint -config-file .gitea/actionlint.yaml -color .gitea/workflows/*.yaml
+
+  lint-shellcheck:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Lint shell scripts with ShellCheck
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh shellcheck)"
+          export PATH="$tools_dir:$PATH"
+          # userbot/ is a git subtree synced from forust/userbot, so its shell
+          # scripts are upstream's to maintain, not ours. Linting them would let a
+          # routine subtree pull turn the deploy gate red on code we do not own.
+          mapfile -t scripts < <(
+            git ls-files '*.sh' ':(glob)**/*.bash' ':!userbot/**'
+          )
+          if [ "${#scripts[@]}" -eq 0 ]; then
+            echo "No shell scripts found."
+            exit 0
+          fi
+          shellcheck --external-sources --source-path=SCRIPTDIR --severity=style "${scripts[@]}"
+
   lint-prettier:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -24,6 +112,10 @@ jobs:
       - name: Check formatting with Prettier
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh prettier)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t prettier_files < <(
             git ls-files \
               | grep -E '\.(md|json|ya?ml|html|css)$' \
@@ -39,17 +131,23 @@ jobs:
 
   lint-ruff:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Lint Python with Ruff
+      - name: Lint and format-check Python with Ruff
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh ruff)"
+          export PATH="$tools_dir:$PATH"
           ruff check .
+          ruff format --check .
 
   lint-yaml:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -57,6 +155,10 @@ jobs:
       - name: Lint YAML syntax
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh yamllint)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t yaml_files < <(
             git ls-files '*.yaml' '*.yml' \
               ':!node_modules/**' \
@@ -72,6 +174,7 @@ jobs:
 
   lint-dockerfiles:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -79,6 +182,10 @@ jobs:
       - name: Lint Dockerfiles
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh hadolint)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t dockerfiles < <(
             git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
           )
@@ -90,15 +197,169 @@ jobs:
 
           hadolint -c .hadolint.yaml "${dockerfiles[@]}"
 
-  validate:
+  # Known, accepted, and recorded. Each line is a real advisory against a
+  # package we build into the panel image, kept in this workflow rather than in
+  # the package manifest so that a subtree sync from forust/userbot cannot
+  # silently widen the exemption.
+  #
+  # starlette is the reason this job is not simply "fail on everything":
+  # fastapi 0.115.12 pins `starlette<0.47.0`, and the fixes for the last four
+  # below need 0.49.1 through 1.3.1, so clearing them means a jump from fastapi
+  # 0.115.12 to 0.141.x. That is upstream's call, not a drive-by in a lint
+  # commit. Of the seven, four are reachable here in principle: 1942 is a
+  # crafted Range header hitting FileResponse, and the panel serves its built
+  # SPA through exactly that; 249 is request.form() ignoring max_fields for
+  # x-www-form-urlencoded, which is the login form; 1941 is a large multipart
+  # body blocking the event loop; 161 and 248 are unvalidated Host and request
+  # path reaching request.url. 2280 needs HTTPEndpoint, which the panel does
+  # not use, and 2281 is Windows-only, and this deploys on Linux.
+  #
+  # The panel answers on userbot.workstation.internal and has no public
+  # forust.xyz route, which is what keeps the four reachable ones from being
+  # an internet-facing DoS. It still manages Telegram credentials.
+  #
+  # Deleting an entry here is how you accept a new advisory, so the diff says
+  # so out loud.
+  scan-deps:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 15
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Validate Kubernetes manifests
+      - name: Audit the Python dependencies that ship in the image
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh pip-audit)"
+          export PATH="$tools_dir:$PATH"
+          # requirements.txt, not requirements-dev.txt: this is what the image
+          # installs, and the test tooling is not a shipped attack surface.
+          pip-audit -r userbot/panel/backend/requirements.txt --strict \
+            --ignore-vuln CVE-2025-67720 \
+            --ignore-vuln PYSEC-2026-161 \
+            --ignore-vuln PYSEC-2026-1941 \
+            --ignore-vuln PYSEC-2026-1942 \
+            --ignore-vuln PYSEC-2026-2280 \
+            --ignore-vuln PYSEC-2026-2281 \
+            --ignore-vuln PYSEC-2026-248 \
+            --ignore-vuln PYSEC-2026-249
+
+      # devDependencies are excluded on purpose. `npm audit` on the full tree
+      # reports 7 findings, and every one of them is a build- or test-time
+      # package: the esbuild CORS advisory needs a vite dev server serving to
+      # the internet, and nanoid's infinite loop needs a custom generator
+      # called with size 0, which postcss does not do. None of them are in the
+      # 91 kB bundle the panel serves. The one production finding, devalue
+      # via svelte, is moderate, which is where --audit-level draws the line;
+      # this fails on the next high or critical one.
+      - name: Audit the production npm dependencies
+        shell: bash
+        run: |
+          set -euo pipefail
+          # The pinned node, not whatever the runner has. Its system node is a
+          # rolling Arch package: during this very push its npm was missing
+          # entirely, and an hour later it was npm 12 on node 26. Both are the
+          # wrong major anyway — the panel image is node:22-alpine.
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh node)"
+          export PATH="$tools_dir:$PATH"
+          cd userbot/panel/frontend
+          npm ci
+          npm audit --omit=dev --audit-level=high
+
+  test-backend:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # 25 tests over the panel's pydantic models, its auth flow, the SPA
+      # fallback and the Kubernetes client it shells out with. They existed and
+      # had never been executed by anything.
+      #
+      # Note that userbot/ is a subtree synced from forust/userbot, so a routine
+      # sync can turn this red on upstream's code. Unlike the shellcheck job,
+      # which skips that tree because style disagreements there are ours to
+      # lose, a failing test here is a real defect in a service we deploy.
+      - name: Run the panel backend test suite
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh uv)"
+          export PATH="$tools_dir:$PATH"
+
+          # A venv in a temp dir rather than a checked-out one: the runner is
+          # shared, and a leftover .venv would let a dependency the
+          # requirements no longer pin still satisfy an import.
+          #
+          # --python is not optional. uv otherwise takes whatever interpreter it
+          # finds first, and which one that is depends on the machine: this
+          # runner runs jobs on the host, where the only interpreter is 3.14,
+          # and pyrogram's sync.py calls the bare asyncio.get_event_loop() that
+          # 3.14 no longer auto-creates, so three tests fail at collection. The
+          # image is python:3.13-slim, so 3.13 is also the version worth
+          # testing: uv fetches a managed build of it when the host has none,
+          # which is what makes this job independent of the runner.
+          venv="$(mktemp -d)/venv"
+          uv venv --python 3.13 --quiet "$venv"
+          uv pip install --quiet --python "$venv/bin/python" \
+            -r userbot/panel/backend/requirements-dev.txt
+
+          # `python -m`, not bare `pytest`: the tests import `app.*` relative to
+          # the backend directory, which only works if the cwd is on sys.path,
+          # and only `python -m` puts it there.
+          cd userbot/panel/backend
+          "$venv/bin/python" -m pytest tests/ -q
+
+  test-frontend:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # One `npm ci` for both checks below: it is by far the slowest part of
+      # this job, and a second one would learn nothing the first did not.
+      #
+      # `npm ci`, not `npm install`, for the same reason the Dockerfile uses it:
+      # the lockfile is what makes the tree that gets checked the tree that
+      # gets shipped.
+      - name: Type-check and test the panel frontend
+        shell: bash
+        run: |
+          set -euo pipefail
+          # The pinned node, not whatever the runner has. Its system node is a
+          # rolling Arch package: during this very push its npm was missing
+          # entirely, and an hour later it was npm 12 on node 26. Both are the
+          # wrong major anyway — the panel image is node:22-alpine.
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh node)"
+          export PATH="$tools_dir:$PATH"
+          cd userbot/panel/frontend
+          npm ci
+
+          # svelte-check has been a devDependency all along with no script
+          # pointing at it, so the type errors it reports had nowhere to
+          # surface. It is clean today, which is the only reason it can be a
+          # gate: it stops at whatever upstream introduces rather than
+          # reporting a backlog we inherited.
+          npm run check
+          npm test
+
+  validate:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Validate Kubernetes manifests against JSON schemas
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t manifests < <(
             git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
               | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
@@ -115,10 +376,115 @@ jobs:
             -summary \
             "${manifests[@]}"
 
+      # kubeconform has no schemas for CRDs, so every IngressRoute, Certificate,
+      # PrometheusRule, Middleware, ServersTransport and ServiceMonitor is silently
+      # skipped above. The live API server knows the real CRD schemas (and runs the
+      # cert-manager / Traefik admission webhooks), so validate there too.
+      #
+      # Only services marked with a k8s/active marker are checked: server-side
+      # dry-run needs the target namespace to exist, and inactive services are not
+      # deployed. Services being enabled for the first time are still covered by
+      # the JSON-schema pass above.
+      #
+      # Main pushes only. `--dry-run=server` persists nothing, but it does execute
+      # the admission webhooks of the production API server, so anyone able to open
+      # a pull request would be able to run arbitrary manifest content through
+      # cert-manager and Traefik. A pull request has nothing to gain from it either:
+      # only main is ever deployed, and this job runs to completion before the
+      # deploy workflow is allowed to start, so a bad CRD is still caught before
+      # anything reaches the cluster -- just on the push rather than on the PR.
+      - name: Note the server-side check is not running here
+        if: github.event_name == 'pull_request' || github.ref != 'refs/heads/main'
+        shell: bash
+        run: |
+          echo "::notice::Skipping the server-side dry-run. It executes the cert-manager and" \
+            "Traefik admission webhooks against the production API server, so it is limited" \
+            "to pushes to main. CRDs are still schema-checked by kubeconform above, and the" \
+            "server-side pass still runs on main before the deploy."
+
+      - name: Validate active manifests against the live API server
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if ! kubectl get --raw='/readyz' --request-timeout=10s >/dev/null 2>&1; then
+            echo "::warning::Cluster unreachable — skipped server-side validation of CRDs (IngressRoute, Certificate, PrometheusRule). Review manifest changes manually."
+            exit 0
+          fi
+
+          mapfile -t k8s_dirs < <(
+            git ls-files '*.yaml' '*.yml' \
+              | grep -E '(^|/)k8s/' \
+              | sed -E 's#((^|.*/)k8s)/.*#\1#' \
+              | sort -u
+          )
+
+          manifests=()
+          kustomize_apps=()
+          for dir in "${k8s_dirs[@]}"; do
+            if [ ! -f "${dir}/active" ]; then
+              echo "skip (no k8s/active): ${dir}"
+              continue
+            fi
+            if [ -f "${dir}/overlays/prod/kustomization.yaml" ]; then
+              kustomize_apps+=("${dir}/overlays/prod")
+            elif [ -f "${dir}/base/kustomization.yaml" ]; then
+              kustomize_apps+=("${dir}/base")
+            else
+              while IFS= read -r f; do
+                [ -n "$f" ] && manifests+=("$f")
+              done < <(
+                git ls-files "${dir}/*.yaml" "${dir}/*.yml" \
+                  | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
+              )
+            fi
+          done
+
+          echo "server-side dry-run: ${#manifests[@]} manifests, ${#kustomize_apps[@]} kustomize apps"
+          failed=0
+          for m in ${manifests[@]+"${manifests[@]}"}; do
+            if ! out="$(kubectl apply --dry-run=server -f "$m" 2>&1)"; then
+              failed=1
+              echo "::error file=${m}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+          for k in ${kustomize_apps[@]+"${kustomize_apps[@]}"}; do
+            if ! out="$(kubectl apply -k "$k" --dry-run=server 2>&1)"; then
+              failed=1
+              echo "::error file=${k}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+
+          if [ "$failed" -ne 0 ]; then
+            echo "Server-side validation failed. The API server (or an admission webhook) rejected these manifests."
+            exit 1
+          fi
+          echo "server-side dry-run: all active manifests accepted by the API server"
+
   build:
-    needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
+    needs:
+      # scan-deps and the two test jobs were missing here, so a commit with a
+      # known-vulnerable dependency or a failing test still moved the :prod tag.
+      # The deploy was blocked either way - it requires the whole workflow to
+      # have succeeded - but the tag had already moved, and the next deploy to
+      # run resolved it. Publishing and passing the checks are the same gate.
+      [
+        lint-actionlint,
+        lint-shellcheck,
+        lint-compose,
+        lint-prettier,
+        lint-ruff,
+        lint-yaml,
+        lint-dockerfiles,
+        scan-deps,
+        test-backend,
+        test-frontend,
+        validate,
+      ]
     if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 60
     outputs:
       services: ${{ steps.services.outputs.services }}
     steps:
@@ -131,12 +497,20 @@ jobs:
         id: services
         shell: bash
         run: |
+          set -euo pipefail
           base="${{ github.event.before }}"
           if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
             base="$(git rev-list --max-parents=0 HEAD)"
           fi
 
-          mapfile -t changed_files < <(git diff --name-only "$base" "${GITHUB_SHA}")
+          # A failed diff used to leave changed_files empty, which reads exactly
+          # like "nothing to build": the job went green having built nothing and
+          # the tag never moved. The status is checked, not assumed.
+          if ! changed="$(git diff --name-only "$base" "${GITHUB_SHA}")"; then
+            echo "::error::cannot diff ${base}..${GITHUB_SHA}"
+            exit 1
+          fi
+          mapfile -t changed_files <<<"$changed"
 
           services=()
 
@@ -186,30 +560,57 @@ jobs:
       - name: Log in to registry
         if: steps.services.outputs.services != ''
         shell: bash
+        # Through env, not by substitution into the script. A secret written
+        # into a run: block is pasted into the shell source before bash parses
+        # it, so a password containing a quote, a backtick or $(...) becomes
+        # code that runs. Masking the value in the log does not prevent that.
+        env:
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
         run: |
-          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-            -u "${{ secrets.REGISTRY_USERNAME }}" \
+          set -euo pipefail
+          printf '%s' "$REGISTRY_PASSWORD" | docker login "${REGISTRY}" \
+            -u "$REGISTRY_USERNAME" \
             --password-stdin
 
       - name: Build and push changed images
         if: steps.services.outputs.services != ''
         shell: bash
         run: |
+          # This step was the one run: block in the workflow without it, and it
+          # is the one that cannot afford it: a docker push that failed partway
+          # through the loop used to be followed by more pushes, the loop's exit
+          # status came from the last one, and the job went green with half the
+          # images missing from the registry.
+          set -euo pipefail
           IFS=, read -r -a services <<< "${{ steps.services.outputs.services }}"
 
+          # Tags for this push. The commit-pinned name is the point of this
+          # step: the deploy resolves it in preference to :prod, so a deploy
+          # that sat in the queue behind a later push still gets the build of
+          # the commit CI validated, instead of whatever :prod points at by the
+          # time it runs. See render_pinned in deploy-lib.sh.
+          commit_tag=""
+          if [ "${GITHUB_REF_NAME}" = "main" ]; then
+            commit_tag="sha-${GITHUB_SHA:0:12}"
+          fi
+
+          set_tags() {
+            tags=()
+            case "${GITHUB_REF_NAME}" in
+              main) tags+=("main" "prod") ;;
+              dev) tags+=("dev") ;;
+            esac
+            if [ -n "$commit_tag" ]; then
+              tags+=("$commit_tag")
+            fi
+          }
+
           for service in "${services[@]}"; do
             case "$service" in
               dtek_notif)
                 image="${REGISTRY}/forust/dtek-notif"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
+                set_tags
                 build_args=()
                 for tag in "${tags[@]}"; do
                   build_args+=(-t "${image}:${tag}")
@@ -224,15 +625,7 @@ jobs:
                 ;;
               errorpages)
                 image="${REGISTRY}/forust/error-pages"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
+                set_tags
                 build_args=()
                 for tag in "${tags[@]}"; do
                   build_args+=(-t "${image}:${tag}")
@@ -246,15 +639,7 @@ jobs:
                 done
                 ;;
               userbot)
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
+                set_tags
                 for target in runtime panel; do
                   case "$target" in
                     runtime)
@@ -280,8 +665,8 @@ jobs:
                 done
                 ;;
               homepages)
-                for service in forust xdfnx; do
-                  case "$service" in
+                for variant in forust xdfnx; do
+                  case "$variant" in
                     forust)
                       image="${REGISTRY}/forust/forust-homepage"
                       ;;
@@ -289,15 +674,7 @@ jobs:
                       image="${REGISTRY}/forust/xdfnx-homepage"
                       ;;
                   esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
+                  set_tags
                   build_args=()
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
@@ -305,15 +682,15 @@ jobs:
                   docker build \
                     --cache-from "type=registry,ref=${image}:buildcache" \
                     --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
-                    "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
+                    "${build_args[@]}" -f "homepages/Dockerfile.${variant}" homepages
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
                 done
                 ;;
               edu_master)
-                for service in session-keeper webinar-checker; do
-                  case "$service" in
+                for variant in session-keeper webinar-checker; do
+                  case "$variant" in
                     session-keeper)
                       context="edu_master/phpsessid-bot"
                       image="${REGISTRY}/forust/session-keeper"
@@ -323,15 +700,7 @@ jobs:
                       image="${REGISTRY}/forust/webinar-checker"
                       ;;
                   esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
+                  set_tags
                   build_args=()
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
@@ -347,3 +716,42 @@ jobs:
                 ;;
             esac
           done
+
+      # Every image the tree names has to carry the commit-pinned name, not only
+      # the ones this push rebuilt. A push that touches nothing but manifests
+      # builds nothing, and its deploy would then find no commit-pinned tag to
+      # resolve and quietly fall back to the moving :prod - which is the whole
+      # failure the commit-pinned name exists to remove.
+      #
+      # Re-tagging copies the manifest list and transfers no layers, so pinning
+      # six images that already exist costs six registry writes.
+      #
+      # The list is derived from the tree rather than written out here, so an
+      # image added to a manifest is covered without a second place to update.
+      - name: Pin the commit name on the images this push did not rebuild
+        if: github.ref_name == 'main'
+        shell: bash
+        run: |
+          set -euo pipefail
+          commit_tag="sha-${GITHUB_SHA:0:12}"
+          mapfile -t repos < <(
+            git grep -hoE 'gcr\.forust\.xyz/forust/[A-Za-z0-9._-]+' -- '*.yaml' '*.yml' \
+              | sort -u
+          )
+          if [ "${#repos[@]}" -eq 0 ]; then
+            echo "No own images referenced by the tree."
+            exit 0
+          fi
+          echo "pinning ${#repos[@]} image(s) to $commit_tag"
+          for repo in "${repos[@]}"; do
+            if docker buildx imagetools inspect "$repo:$commit_tag" >/dev/null 2>&1; then
+              echo "  already built by this push: ${repo##*/}"
+              continue
+            fi
+            if ! docker buildx imagetools inspect "$repo:prod" >/dev/null 2>&1; then
+              echo "  WARNING: ${repo##*/} has no :prod to pin and no build produced it"
+              continue
+            fi
+            docker buildx imagetools create --tag "$repo:$commit_tag" "$repo:prod"
+            echo "  pinned ${repo##*/}"
+          done

.gitea/workflows/compose-lint.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Shared helpers for validating Compose files. Sourced both by steps in
+# .gitea/workflows/ci.yaml and by deploy-lib.sh on the workstation.
+#
+# Two levels of checking, matching how the repo is structured:
+#
+#   general  every committed Compose file, active or not. Pure structure check:
+#            no ${VAR} interpolation, no .env lookup, no bind-mount path
+#            resolution. Disabled stacks deliberately have no .env in the repo
+#            and no values on the CI runner, so a full `config` run would fail on
+#            their `${VAR:?}` guards for reasons that have nothing to do with the
+#            change under review.
+#
+#   full     active stacks only, with interpolation and env-file resolution, so
+#            required variables and referenced files are actually resolved. Needs
+#            the gitignored .env files, so this only runs in the deploy workflow
+#            on the workstation.
+#
+# This file is meant to be sourced, not executed.
+
+# All committed Compose files, including the ones deploy never starts.
+compose_files() {
+  git ls-files \
+    '*/compose.yaml' '*/compose.yml' 'compose.yaml' 'compose.yml' \
+    '*/docker-compose.yaml' '*/docker-compose.yml'
+}
+
+# Prints the flags that turn `docker compose config` into the general check.
+# Probed rather than hardcoded so an older Compose without --no-env-resolution
+# still gets the flags it does support.
+compose_safe_flags() {
+  local help flag
+  help="$(docker compose config --help 2>/dev/null || true)"
+  for flag in --no-interpolate --no-env-resolution --no-path-resolution; do
+    if printf '%s' "$help" | grep -q -- "$flag"; then
+      printf '%s\n' "$flag"
+    fi
+  done
+}
+
+# validate_compose_file <file> [extra docker compose config flags...]
+validate_compose_file() {
+  local file="$1"
+  shift
+  docker compose -f "$file" config --quiet "$@"
+}

.gitea/workflows/deploy.yaml

@@ -1,13 +1,26 @@
 name: deploy
 
 on:
-  push:
-    branches:
-      - main
+  # Deploy only what CI already validated. workflow_run is used instead of
+  # workflow_dispatch so a red lint/validate run can never reach the cluster.
+  workflow_run:
+    workflows: [ci]
+    types: [completed]
   workflow_dispatch:
 
+# The deploy jobs read the tree, then reach the cluster over SSH with the
+# deploy key. The Actions token itself is not part of that path, so it gets
+# read-only contents and no more.
+permissions:
+  contents: read
+
 concurrency:
   group: deploy-main
+  # Queue instead of cancelling. Cancelling a run kills the apply job mid-loop and
+  # takes the verify job down with it, so a superseded deploy would leave the
+  # cluster half-applied and unchecked — the exact failure the verify job exists
+  # to catch. kubectl apply and docker compose up are both idempotent, so letting
+  # the older run finish and then deploying the newer commit costs little.
   cancel-in-progress: false
 
 env:
@@ -17,10 +30,21 @@ env:
   DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
   DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
   APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
+  # workflow_run's own GITHUB_SHA points at the branch head, not at the commit the
+  # finished ci run checked. Pin the exact validated commit instead, so a push
+  # landing mid-deploy cannot make the workstation deploy something else. Also
+  # what the verify job checks the snapshot against. Empty for workflow_dispatch,
+  # which falls back to the current origin/main.
+  DEPLOY_SHA: ${{ github.event.workflow_run.head_sha }}
 
 jobs:
   preflight:
+    if: >-
+      github.event_name != 'workflow_run' ||
+      (github.event.workflow_run.conclusion == 'success' &&
+       github.event.workflow_run.head_branch == 'main')
     runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -34,6 +58,7 @@ jobs:
   validate:
     needs: [preflight]
     runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -47,6 +72,36 @@ jobs:
   apply-k8s:
     needs: [validate]
     runs-on: [self-hosted, linux, arch, homelab, prod]
+    # Apply only, no verification, so this is just the work itself: snapshot,
+    # then sequential `helm upgrade --atomic --timeout 10m`, then the apply loop.
+    # Verification has its own job and its own budget.
+    #
+    # 45 is roughly four times the measured cost of the stage, which is
+    # deliberately not raised on a theory:
+    #
+    #   helm, healthy          3 no-op upgrades              ~3-5 min
+    #   helm, one release bad  --atomic spends its 10m,      ~10-15 min
+    #                            then rolls that one back
+    #   apply loop             ~40 manifests, 4 of which     ~1 min
+    #                           resolve an image digest
+    #   restart_stale_images   7.6s to find 8 workloads,     ~0.5 min
+    #                           9.8s to resolve their digests
+    #
+    # The helm figure is one release, not three: `set -e` aborts
+    # upgrade_helm_releases on the first failure, so a broken release costs
+    # 10m and the other two are never attempted. Multiplying 10m by three
+    # overstates the worst case by 20 minutes.
+    #
+    # The 45 minutes this was last raised to 45 were still not enough, and the
+    # job logs for those runs no longer exist, so what actually consumed the
+    # budget is not known - the two measurable candidates above account for
+    # ~15 of it. The one unbounded thing left in this stage is
+    # `docker manifest inspect` at deploy-lib.sh:236, which has no timeout
+    # against a registry with a known hang mode. Bound it, and make the stage
+    # announce what it is working on, before spending any of that on a larger
+    # ceiling: a stage that is killed with a diagnosable last line is a bug
+    # report, one that vanishes is not.
+    timeout-minutes: 45
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -60,6 +115,7 @@ jobs:
   apply-compose:
     needs: [validate]
     runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -69,3 +125,68 @@ jobs:
         run: |
           set -euo pipefail
           ./.gitea/workflows/ssh-run.sh apply-compose
+
+  # Watches the workloads this deploy changed and rolls back the ones that never
+  # became healthy. Runs even when the apply jobs failed, timed out or were
+  # cancelled — that is the whole point of splitting it out. `always()` is what
+  # lets it start after a failed dependency; the needs on apply-compose are a
+  # barrier, so verification begins only once both applies are done.
+  verify-k8s:
+    needs: [apply-k8s, apply-compose]
+    if: >-
+      always() &&
+      needs.apply-k8s.result != 'skipped' &&
+      needs.apply-compose.result != 'skipped'
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    # Not raised, because the arithmetic does not close.
+    #
+    # 32 workloads are under management and the wave width is 8, so the verify
+    # itself is 4 waves of ROLLOUT_TIMEOUT (300s) = 20 minutes worst case, when
+    # every rollout times out rather than converging. That is already 20 of 30.
+    #
+    # The other 10 would have to absorb rollback, and rollback_workloads is a
+    # serial `while read` loop at 300s per failed workload. 10 minutes buys two.
+    # Any larger number is buying a bigger multiple of an unbounded term rather
+    # than covering a known cost: 60 minutes buys eight, and 60 minutes is
+    # therefore not a bound, it is a guess with two digits.
+    #
+    # The number becomes derivable the moment rollback uses the same wave width
+    # as the verify: 32 failures then cost 4 waves = 20 minutes instead of 160,
+    # and 45 covers verify plus rollback at full width. That change is to the
+    # recovery path and is not folded into a timeout edit.
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Verify workloads and roll back on failure
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh verify-k8s
+
+  # Asks the public route of every active service whether it is actually
+  # serving, which the rollout check above structurally cannot: a pod can
+  # converge and still be crash-looping, or be listening on a port no Service
+  # points at, or answer 500.
+  #
+  # `always()` for the same reason verify-k8s has it, and it runs after that job
+  # specifically because a rollback is when a route most needs re-checking. The
+  # needs is a barrier, not a filter: whether verify-k8s passed, failed or was
+  # cancelled, the probes are what say whether the cluster is serving, and
+  # suppressing them on a rollback would hide the one run where the answer
+  # matters most.
+  smoke:
+    needs: [verify-k8s]
+    if: always() && needs.verify-k8s.result != 'skipped'
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Probe the public route of every active service
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh smoke

.gitea/workflows/install-ci-tools.sh

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+# Installs the pinned CI tools into "$TOOLS_DIR/bin" and echoes that directory
+# on stdout, so callers can do:
+#
+#   export PATH="$(bash .gitea/workflows/install-ci-tools.sh kubeconform shellcheck):$PATH"
+#
+# Versions come from tool-versions.env next to this script and are kept fresh by
+# Renovate. Re-running is cheap: an already-installed tool at the pinned version
+# is left alone.
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tool-versions.env
+. "$here/tool-versions.env"
+
+TOOLS_DIR="${TOOLS_DIR:-${RUNNER_TEMP:-/tmp}/homelab-tools}"
+BIN_DIR="$TOOLS_DIR/bin"
+mkdir -p "$BIN_DIR"
+
+arch="$(uname -m)"
+# Upstream projects disagree on arch spelling: kubeconform and actionlint use
+# Go names (amd64/arm64), shellcheck uses uname names (x86_64/aarch64), node
+# uses neither (x64/arm64), and hadolint mixes the two in a single release
+# (x86_64 but arm64).
+case "$arch" in
+  x86_64 | amd64)
+    goarch=amd64
+    sharch=x86_64
+    nodearch=x64
+    hadolintarch=x86_64
+    ;;
+  aarch64 | arm64)
+    goarch=arm64
+    sharch=aarch64
+    nodearch=arm64
+    hadolintarch=arm64
+    ;;
+  *)
+    echo "install-ci-tools: unsupported architecture: $arch" >&2
+    exit 1
+    ;;
+esac
+
+fetch() {
+  # fetch <url> <dest>
+  if command -v curl >/dev/null 2>&1; then
+    curl -sSLf --retry 3 -o "$2" "$1"
+  elif command -v wget >/dev/null 2>&1; then
+    wget -q -O "$2" "$1"
+  else
+    echo "install-ci-tools: neither curl nor wget is available" >&2
+    exit 1
+  fi
+}
+
+# installed_version <command>
+# Prints the version of an already-installed tool, or nothing. Each tool spells
+# its version flag differently, hence the case.
+installed_version() {
+  local out
+  case "$1" in
+    kubeconform) out="$("$1" -v 2>/dev/null | head -1 || true)" ;;
+    *) out="$("$1" --version 2>/dev/null | head -1 || true)" ;;
+  esac
+  printf '%s' "$out"
+}
+
+# at_version <command> <expected>
+at_version() {
+  case "$(installed_version "$1")" in
+    *"$2"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+install_kubeconform() {
+  if at_version kubeconform "v${KUBECONFORM_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${goarch}.tar.gz" \
+    "$tmp/kubeconform.tar.gz"
+  tar -xzf "$tmp/kubeconform.tar.gz" -C "$tmp" kubeconform
+  install -m 0755 "$tmp/kubeconform" "$BIN_DIR/kubeconform"
+  rm -rf "$tmp"
+}
+
+install_shellcheck() {
+  if at_version shellcheck "${SHELLCHECK_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.${sharch}.tar.xz" \
+    "$tmp/shellcheck.tar.xz"
+  tar -xJf "$tmp/shellcheck.tar.xz" -C "$tmp" --strip-components=1 "shellcheck-v${SHELLCHECK_VERSION}/shellcheck"
+  install -m 0755 "$tmp/shellcheck" "$BIN_DIR/shellcheck"
+  rm -rf "$tmp"
+}
+
+install_uv() {
+  if at_version uv "${UV_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  # uv release tags carry no leading v, unlike every other tool installed here.
+  fetch "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-${sharch}-unknown-linux-gnu.tar.gz" \
+    "$tmp/uv.tar.gz"
+  tar -xzf "$tmp/uv.tar.gz" -C "$tmp" --strip-components=1 "uv-${sharch}-unknown-linux-gnu/uv"
+  install -m 0755 "$tmp/uv" "$BIN_DIR/uv"
+  rm -rf "$tmp"
+}
+
+install_hadolint() {
+  if at_version hadolint "${HADOLINT_VERSION}"; then
+    return 0
+  fi
+  # A bare binary, no archive: hadolint ships one file per platform.
+  fetch "https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VERSION}/hadolint-linux-${hadolintarch}" \
+    "$BIN_DIR/hadolint"
+  chmod 0755 "$BIN_DIR/hadolint"
+}
+
+# ruff and yamllint both come from PyPI as wheels, which uv unpacks for us.
+install_uv_tool() {
+  # <package> <pinned version>
+  if at_version "$1" "$2"; then
+    return 0
+  fi
+  install_uv
+  UV_TOOL_BIN_DIR="$BIN_DIR" uv tool install --force "$1==$2" >/dev/null
+}
+
+install_ruff() {
+  install_uv_tool ruff "${RUFF_VERSION}"
+}
+
+install_yamllint() {
+  install_uv_tool yamllint "${YAMLLINT_VERSION}"
+}
+
+install_pip_audit() {
+  install_uv_tool pip-audit "${PIP_AUDIT_VERSION}"
+}
+
+install_prettier() {
+  if at_version prettier "${PRETTIER_VERSION}"; then
+    return 0
+  fi
+  # Not a standalone binary: prettier's entry point requires ../package.json
+  # relative to its own real path, so the package directory has to survive
+  # next to it. Hence a versioned directory plus a relative symlink, rather
+  # than copying the one file out as the other installers do.
+  local dir="$BIN_DIR/prettier-${PRETTIER_VERSION}"
+  if [ ! -f "$dir/package/package.json" ]; then
+    rm -rf "$dir"
+    mkdir -p "$dir"
+    fetch "https://registry.npmjs.org/prettier/-/prettier-${PRETTIER_VERSION}.tgz" "$dir/prettier.tgz"
+    tar -xzf "$dir/prettier.tgz" -C "$dir"
+    rm -f "$dir/prettier.tgz"
+    # npm strips the exec bit from bin/ on the way into the tarball.
+    chmod 0755 "$dir/package/bin/prettier.cjs"
+  fi
+  # Relative, so the whole tree stays valid if TOOLS_DIR is relocated.
+  ln -sfn "prettier-${PRETTIER_VERSION}/package/bin/prettier.cjs" "$BIN_DIR/prettier"
+}
+
+install_node() {
+  # npm gets checked by running it, not by looking it up: what matters is that
+  # it answers, so a stub, a half-removed Arch package or a name that resolves
+  # to something broken all have to read as "not installed". The runner's npm
+  # is a symlink into /usr/lib/node_modules/npm, which is exactly the kind of
+  # thing that disappears between runs.
+  if at_version node "v${NODE_VERSION}" && [ -n "$(installed_version npm)" ]; then
+    return 0
+  fi
+  # Same shape as prettier above: the tarball's bin/npm and bin/npx are links
+  # into lib/node_modules, so the whole tree has to survive next to them.
+  local dir="$BIN_DIR/node-${NODE_VERSION}"
+  if [ ! -x "$dir/bin/node" ]; then
+    rm -rf "$dir"
+    mkdir -p "$dir"
+    fetch "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${nodearch}.tar.xz" \
+      "$dir/node.tar.xz"
+    tar -xJf "$dir/node.tar.xz" -C "$dir" --strip-components=1 "node-v${NODE_VERSION}-linux-${nodearch}"
+    rm -f "$dir/node.tar.xz"
+  fi
+  # Relative, so the whole tree stays valid if TOOLS_DIR is relocated.
+  for bin in node npm npx; do
+    ln -sfn "node-${NODE_VERSION}/bin/${bin}" "$BIN_DIR/${bin}"
+  done
+}
+
+install_actionlint() {
+  if at_version actionlint "${ACTIONLINT_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_${goarch}.tar.gz" \
+    "$tmp/actionlint.tar.gz"
+  tar -xzf "$tmp/actionlint.tar.gz" -C "$tmp" actionlint
+  install -m 0755 "$tmp/actionlint" "$BIN_DIR/actionlint"
+  rm -rf "$tmp"
+}
+
+wanted=("$@")
+if [ "${#wanted[@]}" -eq 0 ]; then
+  wanted=(kubeconform shellcheck actionlint prettier ruff yamllint hadolint)
+fi
+
+for tool in "${wanted[@]}"; do
+  case "$tool" in
+    kubeconform) install_kubeconform ;;
+    shellcheck) install_shellcheck ;;
+    actionlint) install_actionlint ;;
+    prettier) install_prettier ;;
+    ruff) install_ruff ;;
+    yamllint) install_yamllint ;;
+    pip-audit) install_pip_audit ;;
+    hadolint) install_hadolint ;;
+    node) install_node ;;
+    uv) install_uv ;;
+    *)
+      echo "install-ci-tools: unknown tool: $tool" >&2
+      exit 1
+      ;;
+  esac
+done
+
+printf '%s\n' "$BIN_DIR"

.gitea/workflows/renovate-ci.yaml

@@ -7,33 +7,58 @@ on:
       - main
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   validate-renovate:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Validate Renovate Compose draft
+      # renovate/k8s/cronjob.yaml is the single source of truth for the image tag,
+      # so the same version that runs in the cluster is the one validated here.
+      - name: Resolve the deployed Renovate image
+        id: image
         shell: bash
         run: |
           set -euo pipefail
-          trap 'rm -f renovate/.env' EXIT
-          printf '%s\n' \
-            'RENOVATE_ENDPOINT=https://gitea.example/api/v1' \
-            'RENOVATE_TOKEN=test-token' \
-            'RENOVATE_REPOSITORIES=forust/homelab' \
-            > renovate/.env
-          docker compose -f renovate/renovate-compose.yaml config --quiet
+          image="$(sed -n 's|.*image:[[:space:]]*\(renovate/renovate:[^[:space:]]*\).*|\1|p' \
+            renovate/k8s/cronjob.yaml | head -1)"
+          if [ -z "$image" ]; then
+            echo "::error::no renovate/renovate image found in renovate/k8s/cronjob.yaml"
+            exit 1
+          fi
+          echo "using $image"
+          echo "image=$image" >> "$GITHUB_OUTPUT"
 
-      - name: Validate Kubernetes manifests
+      - name: Validate Renovate repository config
         shell: bash
         run: |
           set -euo pipefail
           docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/yannh/kubeconform:latest \
+            -v "$PWD/renovate:/opt/renovate:ro" \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
+            "${{ steps.image.outputs.image }}" \
+            renovate-config-validator /opt/renovate/renovate.json
+
+      # The CronJob cannot read the repository, so renovate/k8s/configmap.yaml
+      # carries an inlined copy of the config. Fail if it no longer matches.
+      - name: Check the generated Renovate ConfigMap
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/sync-renovate-configmap.sh --check
+
+      - name: Validate Renovate Kubernetes manifests
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
+          export PATH="$tools_dir:$PATH"
+          kubeconform \
             -strict \
             -ignore-missing-schemas \
             -summary \
@@ -41,12 +66,11 @@ jobs:
             renovate/k8s/configmap.yaml \
             renovate/k8s/cronjob.yaml
 
-      - name: Validate Renovate repository config
+      - name: Validate Renovate Compose file
         shell: bash
         run: |
           set -euo pipefail
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            renovate/renovate:44.103.0 \
-            renovate-config-validator renovate.json
+          source .gitea/workflows/compose-lint.sh
+          mapfile -t safe_flags < <(compose_safe_flags)
+          validate_compose_file renovate/renovate-compose.yaml \
+            ${safe_flags[@]+"${safe_flags[@]}"}

.gitea/workflows/renovate-run.yaml

@@ -21,6 +21,11 @@ on:
         default: false
         type: boolean
 
+# Renovate writes through its own bot PAT, passed in as RENOVATE_TOKEN, so the
+# Actions token is only ever used to read the checkout.
+permissions:
+  contents: read
+
 concurrency:
   group: renovate-run
   cancel-in-progress: false
@@ -28,18 +33,36 @@ concurrency:
 jobs:
   run-renovate:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 60
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
+      # renovate/k8s/cronjob.yaml is the single source of truth for the image tag.
+      # Reading it here means this workflow validates and runs the exact version
+      # that is deployed, instead of a copy that silently goes stale.
+      - name: Resolve the deployed Renovate image
+        id: image
+        shell: bash
+        run: |
+          set -euo pipefail
+          image="$(sed -n 's|.*image:[[:space:]]*\(renovate/renovate:[^[:space:]]*\).*|\1|p' \
+            renovate/k8s/cronjob.yaml | head -1)"
+          if [ -z "$image" ]; then
+            echo "::error::no renovate/renovate image found in renovate/k8s/cronjob.yaml"
+            exit 1
+          fi
+          echo "using $image"
+          echo "image=$image" >> "$GITHUB_OUTPUT"
+
       - name: Validate Renovate config
         shell: bash
         run: |
           set -euo pipefail
           docker run --rm \
-            -v "$PWD/renovate/config.js:/opt/renovate/config.js:ro" \
-            -e RENOVATE_CONFIG_FILE=/opt/renovate/config.js \
-            renovate/renovate:44.103.0 \
+            -v "$PWD/renovate/renovate.json:/opt/renovate/renovate.json:ro" \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
+            "${{ steps.image.outputs.image }}" \
             renovate-config-validator
 
       - name: Run Renovate
@@ -56,14 +79,14 @@ jobs:
           : "${RENOVATE_TOKEN:?missing RENOVATE_TOKEN secret — add a renovate-bot PAT in repo/org Actions secrets}"
 
           docker run --rm \
-            -v "$PWD/renovate/config.js:/opt/renovate/config.js:ro" \
+            -v "$PWD/renovate/renovate.json:/opt/renovate/renovate.json:ro" \
             -e RENOVATE_PLATFORM=gitea \
             -e RENOVATE_ENDPOINT=https://gitea.forust.xyz/api/v1 \
             -e RENOVATE_TOKEN="$RENOVATE_TOKEN" \
             -e RENOVATE_GITHUB_COM_TOKEN="${RENOVATE_GITHUB_COM_TOKEN:-}" \
             -e RENOVATE_REPOSITORIES="${RENOVATE_REPOSITORIES:-forust/homelab}" \
             -e RENOVATE_DRY_RUN="${RENOVATE_DRY_RUN:-}" \
-            -e RENOVATE_CONFIG_FILE=/opt/renovate/config.js \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
             -e RENOVATE_BASE_DIR=/tmp/renovate \
             -e LOG_LEVEL="${LOG_LEVEL:-info}" \
-            renovate/renovate:44.103.0
+            "${{ steps.image.outputs.image }}"

.gitea/workflows/ssh-run.sh

@@ -11,15 +11,49 @@ deploy_port="${DEPLOY_PORT:-22}"
 deploy_path="${DEPLOY_PATH:-/srv/homelab}"
 deploy_path="$(printf '%s' "$deploy_path" | tr -d '\"' | tr -d '\r' | xargs)"
 
-ssh_key="$RUNNER_TEMP/deploy_key"
-mkdir -p "$RUNNER_TEMP"
+# The private key is written to a per-run directory that is removed on exit, so a
+# failed or cancelled job cannot leave deploy credentials in the runner's temp
+# directory. Do not use a fixed path: apply-k8s and apply-compose run in parallel.
+key_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/homelab-deploy-key.XXXXXXXX")"
+trap 'rm -rf "$key_dir"' EXIT INT TERM
+
+ssh_key="$key_dir/deploy_key"
 printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
 chmod 600 "$ssh_key"
 
-ssh -i "$ssh_key" -p "$deploy_port" \
-  -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
-  "${DEPLOY_USER}@${DEPLOY_HOST}" \
-  "REPO=$deploy_path APPLY_PRUNE=${APPLY_PRUNE:-false} STAGE=$1 bash -se" <<'EOF'
+# A connection that died silently used to hang until the job timeout, and the
+# stage was never re-run: one flaky TCP session cost a whole 45-minute apply.
+# ServerAlive* bounds how long a dead peer goes unnoticed, ConnectTimeout bounds
+# setup. Only exit 255 - ssh's own transport failures - is retried. A stage that
+# fails on its own merits exits with the remote's status, so a real failure
+# still surfaces its own log instead of burning three attempts. The stages are
+# declarative applies, so re-running one that had already committed is harmless.
+ssh_opts=(
+  -i "$ssh_key" -p "$deploy_port"
+  -o BatchMode=yes -o StrictHostKeyChecking=accept-new
+  -o ConnectTimeout=15
+  -o ServerAliveInterval=15 -o ServerAliveCountMax=4
+)
+
+rc=0
+for attempt in 1 2 3; do
+  if [ "$attempt" -gt 1 ]; then
+    echo ":: warning::ssh transport failed, retrying (${attempt}/3)"
+    sleep $((attempt * 5))
+  fi
+  rc=0
+  ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
+    env "REPO=$deploy_path" "APPLY_PRUNE=${APPLY_PRUNE:-false}" \
+    "DEPLOY_SHA=${DEPLOY_SHA:-}" "DEPLOY_SNAPSHOT_DIR=${DEPLOY_SNAPSHOT_DIR:-}" \
+    "STAGE=$1" bash -se <<'EOF' || rc=$?
 source "$REPO/.gitea/workflows/deploy-lib.sh"
 run_stage "$STAGE"
 EOF
+  [ "$rc" -eq 0 ] && break
+  [ "$rc" -ne 255 ] && break
+done
+
+if [ "$rc" -ne 0 ]; then
+  echo ":: error::stage $1 failed over ssh (exit $rc)"
+fi
+exit "$rc"

.gitea/workflows/sync-renovate-configmap.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Regenerates renovate/k8s/configmap.yaml from renovate/renovate.json.
+#
+# renovate/renovate.json is the single source of truth: the CronJob, the Compose
+# file and the renovate-run workflow all mount that exact file. A ConfigMap cannot
+# read a file from the repository, so the same bytes are inlined here as a literal
+# block. This script keeps the copy honest:
+#
+#   .gitea/workflows/sync-renovate-configmap.sh          # rewrite in place
+#   .gitea/workflows/sync-renovate-configmap.sh --check  # fail if out of date
+#
+# renovate-ci runs the --check form on every PR and push, so a config change that
+# forgets to regenerate the ConfigMap cannot be merged.
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo="$(git -C "$here" rev-parse --show-toplevel)"
+
+src="$repo/renovate/renovate.json"
+dst="$repo/renovate/k8s/configmap.yaml"
+[ -f "$src" ] || {
+  echo "missing $src" >&2
+  exit 1
+}
+
+render() {
+  cat <<'HEADER'
+# GENERATED FILE - do not edit by hand.
+# Source: renovate/renovate.json
+# Regenerate: .gitea/workflows/sync-renovate-configmap.sh
+# Verify:     .gitea/workflows/sync-renovate-configmap.sh --check
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: renovate-config
+  namespace: renovate
+data:
+  renovate.json: |
+HEADER
+  sed 's/^/    /' "$src"
+}
+
+if [ "${1:-}" = "--check" ]; then
+  if ! diff -u "$dst" <(render) >/dev/null 2>&1; then
+    echo "ERROR: $dst is out of sync with renovate/renovate.json"
+    echo "Run: .gitea/workflows/sync-renovate-configmap.sh"
+    diff -u "$dst" <(render) || true
+    exit 1
+  fi
+  echo "renovate/k8s/configmap.yaml is in sync with renovate/renovate.json"
+  exit 0
+fi
+
+render >"$dst"
+echo "wrote $dst"

.gitea/workflows/tool-versions.env

@@ -0,0 +1,33 @@
+# Pinned versions of the CI tools installed by install-ci-tools.sh.
+# Renovate keeps these up to date (see customManagers in renovate/renovate.json).
+#
+# Every version here except NODE_VERSION matches what was already installed on
+# the runner, so pinning them changes what CI does not at all. It changes what
+# CI does when the runner is rebuilt with something else: today
+# install-ci-tools.sh finds the pinned version already on PATH and installs
+# nothing, and a runner that drifts gets the pinned one installed over it.
+#
+# The renovate image version is NOT pinned here: renovate/k8s/cronjob.yaml is the
+# single source of truth and the workflows read the tag from it, so there is
+# nothing to drift.
+ACTIONLINT_VERSION="1.7.7"
+SHELLCHECK_VERSION="0.11.0"
+KUBECONFORM_VERSION="0.8.0"
+PRETTIER_VERSION="3.8.1"
+RUFF_VERSION="0.16.8"
+YAMLLINT_VERSION="1.38.0"
+HADOLINT_VERSION="2.14.0"
+# pip-audit reads the advisory database over the network, so a floating version
+# would make the same commit report different things on different days. Pin it
+# like the rest: the advisories themselves are the moving part, not the tool.
+PIP_AUDIT_VERSION="2.10.1"
+# uv builds the throwaway venv the pytest job runs in, and unpacks the PyPI
+# wheels for ruff, yamllint and pip-audit.
+UV_VERSION="0.12.17"
+# node runs `npm ci` for the frontend tests and the npm audit, and it is the one
+# pin here that does NOT come from the runner: the runner's system node is a
+# rolling Arch package (it was node 26 with no npm at all when this was pinned),
+# and the panel image is node:22-alpine. Pinned to the image's major on purpose,
+# so the tree that gets tested is the tree that gets built. Renovate keeps this
+# in step with the Dockerfile's node: tag via the "node runtime" group.
+NODE_VERSION="22.23.3"

.gitignore

@@ -96,6 +96,8 @@ replacements.txt
 # Temp files
 edu_master/temp/
 temp/*
+# Local-only tooling scratch space (pinned CI tools, verification scripts)
+tmp/
 
 # Environment
 .env

adguardhome/k8s/adguard.yaml

@@ -73,7 +73,7 @@ spec:
               memory: "1.5Gi"
               cpu: "300m"
             requests:
-              memory: "500Mi"
+              memory: "1Gi"
               cpu: "50m"
           ports:
             - containerPort: 3000

authentik/k8s/authentik.yaml

@@ -52,7 +52,7 @@ spec:
             - containerPort: 9000
           resources:
             requests:
-              memory: "700Mi"
+              memory: "768Mi"
               cpu: "300m"
             limits:
               memory: "1.5Gi"
@@ -86,8 +86,8 @@ spec:
                 name: authentik-secrets
           resources:
             requests:
-              memory: "512Mi"
+              memory: "320Mi"
               cpu: "300m"
             limits:
-              memory: "1Gi"
+              memory: "768Mi"
               cpu: "700m"

cfddns/k8s/deployment.yaml

@@ -22,10 +22,10 @@ spec:
           imagePullPolicy: Always
           resources:
             requests:
-              memory: "20Mi"
+              memory: "32Mi"
               cpu: "30m"
             limits:
-              memory: "64Mi"
+              memory: "128Mi"
               cpu: "50m"
           envFrom:
             - secretRef:

cloudflared/k8s/deployment.yaml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: cloudflared
-          image: cloudflare/cloudflared:2026.9.1
+          image: cloudflare/cloudflared:2026.9.3
           imagePullPolicy: IfNotPresent
           args:
             - tunnel
@@ -30,8 +30,8 @@ spec:
                   key: TUNNEL_TOKEN
           resources:
             requests:
-              memory: "32Mi"
+              memory: "128Mi"
               cpu: "30m"
             limits:
-              memory: "128Mi"
+              memory: "256Mi"
               cpu: "200m"

converters/compose.yaml

@@ -54,7 +54,7 @@ services:
       - "traefik.http.routers.bentopdf.tls.certresolver=letsencrypt"
       - "traefik.http.routers.bentopdf.tls=true"
       # Local router
-      - "traefik.http.routers.bentopdf-local.rule=Host(`pdf.wokstation.internal`)"
+      - "traefik.http.routers.bentopdf-local.rule=Host(`pdf.workstation.internal`)"
       - "traefik.http.routers.bentopdf-local.entrypoints=websecure"
       - "traefik.http.routers.bentopdf-local.tls=true"
       # Dev router

converters/k8s/bentopdf.yaml

@@ -31,12 +31,13 @@ spec:
           name: bentopdf
           ports:
             - containerPort: 8080
+          # p95 4M, max 11M over 7 days. Was 50Mi/700Mi.
           resources:
             requests:
-              memory: "50Mi"
+              memory: "32Mi"
               cpu: "50m"
               ephemeral-storage: "100Mi"
             limits:
-              memory: "700Mi"
+              memory: "128Mi"
               cpu: "700m"
               ephemeral-storage: "5Gi"

converters/k8s/convertx.yaml

@@ -38,13 +38,14 @@ spec:
           volumeMounts:
             - mountPath: /data
               name: data
+          # p95 85M, max 136M over 7 days, spikes while converting. Was 250Mi/1.5Gi.
           resources:
             requests:
-              memory: "250Mi"
+              memory: "128Mi"
               cpu: "100m"
             limits:
               cpu: "1500m"
-              memory: "1.5Gi"
+              memory: "512Mi"
       volumes:
         - name: data
           persistentVolumeClaim:

crowdsec/k8s/crowdsec-middleware.yaml

@@ -1,3 +1,17 @@
+# crowdsec/k8s is NOT managed by deploy.yaml - apply this by hand, and apply it
+# together with a restart:
+#   kubectl apply -f crowdsec/k8s/crowdsec-middleware.yaml
+#   kubectl -n traefik rollout restart deploy/traefik
+#
+# The restart is not optional. In stream mode the plugin runs a package-level
+# ticker goroutine (handleStreamTicker over the isCrowdsecStreamHealthy and
+# updateFailure globals) that no reconfiguration stops. Applying a change
+# wedges the instance: every route referencing it answers 404 and traefik logs
+# 'invalid middleware crowdsec-crowdsec-bouncer@kubernetescrd' until the pod is
+# replaced. Re-applying the previous config does NOT recover it, and the config
+# is not the cause - a valid CIDR cannot fail NewChecker, which is a plain
+# net.ParseCIDR. Only a new pod clears it. Measured cost: ~35s down for all
+# 20 hosts behind this middleware.
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
@@ -8,7 +22,48 @@ spec:
     crowdsec-bouncer:
       enabled: true
       LogLevel: INFO
-      CrowdsecMode: live
+      # `live` blocked on a `GET /v1/decisions` per request, so a burst
+      # saturated the LAPI and the plugin 403'd IPs that were never banned.
+      # v1.3.3 ignores UpdateMaxFailure in `live`, so fail-open is only
+      # reachable in stream mode, which polls into a cache instead - no
+      # per-request call to saturate. 15s rather than the 60s default: the
+      # deploy runner shares one public IP with the house, so this bounds
+      # both how late a ban lands and how long a lifted one lingers.
+      CrowdsecMode: stream
+      UpdateIntervalSeconds: 15
+      # -1 = never block because the LAPI is unreachable. In v1.3.3
+      # handleStreamTicker only clears isCrowdsecStreamHealthy when
+      # updateMaxFailure != -1, and ServeHTTP 403s once it is false, so this
+      # makes a CrowdSec outage mean "no protection", not "every site 403".
+      UpdateMaxFailure: -1
       CrowdsecLapiScheme: http
       CrowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080
       CrowdsecLapiKeyFile: "/etc/traefik/secrets/traefik-api-key"
+      # Bypasses the bouncer and the decision cache, no LAPI round-trip.
+      # Keep in sync with forust/local-network in crowdsec-values.yaml.
+      ClientTrustedIPs:
+        - "127.0.0.0/8"
+        - "10.0.0.0/8"
+        - "172.16.0.0/12"
+        - "192.168.0.0/16"
+        - "100.64.0.0/10"
+        - "169.254.0.0/16"
+        - "fc00::/7"
+        - "fe80::/10"
+        # The mobile operator range from forust/mobile-whitelist, repeated
+        # deliberately rather than relying on the parser whitelist alone.
+        # That whitelist drops the event before it reaches a bucket, so no
+        # decision is ever created - but it is one config away from not
+        # firing, and the bouncer would then enforce a ban that was never
+        # justified. This is the last line: even a decision that exists for
+        # any reason is not served against the phone.
+        - "84.245.64.0/18"
+      # The name is HTTPTimeoutSeconds, an int in seconds (min 1) - there is
+      # no CrowdsecLapiTimeout, and an unrecognised key is silently dropped,
+      # which is how this sat at the 10s default. Nothing rides on it per
+      # request any more, so this only bounds the stream pull - and too low
+      # is the dangerous direction: the LAPI needs ~2s to answer
+      # /v1/decisions/stream, and a pull that times out leaves the ban cache
+      # frozen at its startup contents ("failed sending new decisions"),
+      # i.e. new bans silently never apply. Keep it above the pull latency.
+      HTTPTimeoutSeconds: 10

crowdsec/k8s/crowdsec-values.yaml

@@ -58,9 +58,37 @@ config:
           reason: "Mobile IP whitelist"
           cidr:
             - "84.245.64.0/18"
+      # CrowdSec's own guidance: CIDR allowlisting belongs at the parser stage.
+      # A parser whitelist discards the event before it reaches a bucket, so
+      # these addresses never produce an overflow and never become a decision.
+      # A postoverflow whitelist is checked only *after* the ban exists, and
+      # the bouncer answers 403 for as long as it does - which is a window we
+      # do not want the deploy sitting in.
+      local-network.yaml: |
+        name: forust/local-network
+        description: "Whitelist loopback, private and VPN networks"
+        whitelist:
+          reason: "Local network"
+          cidr:
+            - "127.0.0.0/8"
+            - "10.0.0.0/8"
+            - "172.16.0.0/12"
+            - "192.168.0.0/16"
+            # CGNAT range (RFC 6598). The workstation and the k0s node live
+            # here on WireGuard, and 100.64.0.0/10 is not covered by the
+            # RFC 1918 blocks above.
+            - "100.64.0.0/10"
+            - "169.254.0.0/16"
+            - "fc00::/7"
+            - "fe80::/10"
 
   postoverflows:
     s01-whitelist:
+      # The one whitelist that has to stay here: resolving a hostname is a
+      # network call, and the docs put expensive lookups in postoverflows on
+      # purpose - it runs only when a bucket actually overflows.
+      # ddns.forust.xyz is the public home address, not a private one, so
+      # forust/local-network does not cover it.
       home-dynamic-ip.yaml: |
         name: forust/home-dynamic-ip
         description: "Whitelist home dynamic IP"
@@ -69,6 +97,59 @@ config:
           expression:
             - evt.Overflow.Alert.Source.IP in LookupHost("ddns.forust.xyz")
 
+  # LAPI-only main config override, merged over config.yaml. NOTE: the
+  # chart's own default for this key is REPLACED, not merged, so its
+  # auto_registration block is repeated verbatim below - drop it and the
+  # agent can no longer register itself.
+  config.yaml.local: |
+    api:
+      server:
+        auto_registration: # Activate if not using TLS for authentication
+          enabled: true
+          token: "${REGISTRATION_TOKEN}" # /!\ Do not modify this variable (auto-generated and handled by the chart)
+          allowed_ranges: # /!\ Make sure to adapt to the pod IP ranges used by your cluster
+            - "127.0.0.1/32"
+            - "192.168.0.0/16"
+            - "10.0.0.0/8"
+            - "172.16.0.0/12"
+        # This homelab has no egress to console.crowdsec.cloud: DNS does
+        # not resolve. The LAPI kept trying anyway ("Signal push: N
+        # signals to push", "capi metrics: sending" every 10s) and each
+        # attempt sat on a resolver timeout WHILE HOLDING A WRITE
+        # TRANSACTION, which is what kept stalling per-request decision
+        # lookups even with WAL enabled. Nothing to share and nothing to
+        # pull - turn the Central API off instead of letting it block the
+        # only database writer we have.
+        online_client:
+          sharing: false
+          pull:
+            community: false
+            blocklists: false
+        disable_usage_metrics_export: true
+    db_config:
+      # SQLite without WAL serialises every reader behind the writer's
+      # rollback journal, and the LAPI writes constantly: the agent pushes
+      # Traefik alerts read from Loki, the metrics collector counts
+      # decisions, the bouncer touches "last pull" on every request.
+      # Symptom: decision lookups taking 10-30s (and a second connection
+      # that could not even open the database) while the LAPI sat at 28m
+      # CPU - the process was blocked in fsync, not computing. Every
+      # bouncer-protected request then blew through the plugin timeout and
+      # fail-closed with 403, on every site at once.
+      # The PVC is local-path-retain (hostPath), not a network share, so
+      # WAL is safe here; the crowdsec docs recommend it for exactly this
+      # ("allowing more concurrency in SQLite that will improve
+      # performances in most scenarios").
+      use_wal: true
+      # Keeps the alert table bounded. At the 5000/7d default the file
+      # reached 54MB in 15 days off the Traefik access log alone, and the
+      # metrics collector counts decisions on a timer; a smaller working
+      # set means fewer full scans. Crowdsec only prunes - SQLite never
+      # shrinks the file, so the size stays until a manual VACUUM.
+      flush:
+        max_items: 1000
+        max_age: 24h
+
 lapi:
   env:
     - name: COLLECTIONS
@@ -90,13 +171,20 @@ lapi:
       enabled: true
       size: 1Gi
       storageClassName: local-path-retain
+  # LAPI answers a blocking /v1/decisions lookup for EVERY bouncer-protected
+  # request (whole Traefik front door), so it is the hot path of the proxy.
+  # At 400m/500Mi it went CPU-throttled and idle lookups measured 1.3-7.4s,
+  # which pushed requests into the bouncer's fail-closed 403.
+  # Single replica on purpose: LAPI is stateful (BoltDB on the `data` PVC,
+  # credentials on the `config` PVC) - two replicas sharing those RWO
+  # volumes would corrupt the decision store. Scale up CPU, not replicas.
   resources:
     limits:
-      cpu: 400m
-      memory: 500Mi
+      cpu: 1500m
+      memory: 1Gi
     requests:
-      cpu: 50m
-      memory: 150Mi
+      cpu: 250m
+      memory: 500Mi
   service:
     type: ClusterIP
   storeLAPICscliCredentialsInSecret: true

crowdsec/k8s/janitor-cronjob.yaml

@@ -32,6 +32,13 @@
 #          on their own - same name + same password);
 #       4. prune bouncer entries idle for 30d.
 #
+# It used to also delete LePresidente/http-generic-403-bf decisions hourly.
+# That was a workaround for the bouncer failing closed on a slow LAPI and
+# 403-ing the deploy runner into a 4h ban. The bouncer now polls decisions
+# into a cache and never blocks on an unreachable LAPI, so it cannot
+# manufacture those 403s any more, and the scenario only fires against real
+# scanners - deleting their decisions hourly was undoing a working ban.
+#
 # Manual apply (crowdsec/k8s is NOT managed by deploy.yaml):
 #   kubectl apply -f crowdsec/k8s/janitor-cronjob.yaml
 # Force a run:

edu_master/compose.yaml

@@ -1,6 +1,6 @@
 services:
   redis:
-    image: redis:8.10.1-alpine
+    image: redis:8.10.2-alpine
     restart: unless-stopped
     volumes:
       - redis-data:/data
@@ -17,7 +17,7 @@ services:
 
   session-keeper:
     build: ./phpsessid-bot
-    image: gcr.forust.xyz/forust/session-keeper:latest
+    image: gcr.forust.xyz/forust/session-keeper:prod
     pull_policy: build
     env_file: .env
     restart: unless-stopped
@@ -33,7 +33,7 @@ services:
 
   webinar-checker:
     build: ./webinar-checker
-    image: gcr.forust.xyz/forust/webinar-checker:latest
+    image: gcr.forust.xyz/forust/webinar-checker:prod
     pull_policy: build
     env_file: .env
     restart: unless-stopped

edu_master/k8s/alerts.yaml

@@ -11,9 +11,14 @@ spec:
       rules:
         # No successful webinar check for 5m (~2-3 missed 2-min checks).
         # Catches: playwright hangs/timeouts, version skew, site changes, hung job.
+        # The last_success > 0 guard is mandatory: checker.py initialises
+        # last_success to 0, so without it `time() - 0` equals the current epoch
+        # and humanizeDuration renders ~20722d on every pod restart. Keep the
+        # duration expression on the left so $value stays the real gap.
         - alert: WebinarCheckerNoSuccessfulCheck
           expr: |
-            (time() - webinar_check_last_success_timestamp_seconds > 300)
+            ((time() - webinar_check_last_success_timestamp_seconds) > 300)
+              and (webinar_check_last_success_timestamp_seconds > 0)
               and (webinar_check_last_run_timestamp_seconds > 0)
           for: 2m
           labels:
@@ -22,6 +27,20 @@ spec:
             summary: "Webinar checker has no successful check for 5m"
             description: "edu-master/webinar-checker: last successful webinar check was {{ $value | humanizeDuration }} ago. Checks are failing or hanging (see consecutive failures alert). Notifications about new webinars are NOT being sent."
 
+        # Checks are running but none has ever succeeded since pod start.
+        # Split out from the rule above so a zeroed gauge never feeds
+        # humanizeDuration.
+        - alert: WebinarCheckerNeverSucceeded
+          expr: |
+            (webinar_check_last_success_timestamp_seconds == 0)
+              and (webinar_check_last_run_timestamp_seconds > 0)
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker has never completed a successful check"
+            description: 'edu-master/webinar-checker: checks have been running for 10m but not one has ever succeeded since the pod started, so every check is failing. Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
+
         # Fast path: 3 consecutive failures (~6+ min at 2-min interval).
         - alert: WebinarCheckerConsecutiveFailures
           expr: |

edu_master/k8s/playwright.yaml

@@ -20,6 +20,15 @@ spec:
           # renovate: datasource=docker depName=mcr.microsoft.com/playwright versioning=docker
           image: mcr.microsoft.com/playwright:v1.56.0-jammy
           imagePullPolicy: IfNotPresent
+          # p95 412M, max 478M over 7 days, no limit before. Request is set at p95
+          # so the pod is not an eviction candidate; the limit stays above 2x the
+          # request because browser page lifetimes are unpredictable.
+          resources:
+            requests:
+              cpu: "200m"
+              memory: "416Mi"
+            limits:
+              memory: "1Gi"
           command:
             - npx
             - -y

edu_master/k8s/redis.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: redis
-          image: redis:8.10.1-alpine
+          image: redis:8.10.2-alpine
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 6379
@@ -28,10 +28,10 @@ spec:
           resources:
             requests:
               cpu: 25m
-              memory: 64Mi
+              memory: 32Mi
             limits:
               cpu: 250m
-              memory: 256Mi
+              memory: 128Mi
           readinessProbe:
             exec:
               command: ["redis-cli", "ping"]

edu_master/k8s/session-keeper.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       initContainers:
         - name: wait-redis
-          image: redis:8.10.1-alpine
+          image: redis:8.10.2-alpine
           command:
             - /bin/sh
             - -ec
@@ -31,18 +31,17 @@ spec:
               echo "redis is ready"
       containers:
         - name: session-keeper
-          image: gcr.forust.xyz/forust/session-keeper:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/session-keeper:prod
           envFrom:
             - secretRef:
                 name: edu-master-secrets
           resources:
             requests:
               cpu: 25m
-              memory: 96Mi
+              memory: 32Mi
             limits:
               cpu: 250m
-              memory: 256Mi
+              memory: 128Mi
           readinessProbe:
             exec:
               command: ["/bin/sh", "-ec", "redis-cli -h redis EXISTS EDU_PHPSESSID | grep -q 1"]

edu_master/k8s/webinar-checker.yaml

@@ -19,7 +19,7 @@ spec:
       # redis healthy -> session-keeper healthy (EXISTS EDU_PHPSESSID) -> playwright started
       initContainers:
         - name: wait-deps
-          image: redis:8.10.1-alpine
+          image: redis:8.10.2-alpine
           command:
             - /bin/sh
             - -ec
@@ -45,12 +45,19 @@ spec:
               echo "playwright ok"
       containers:
         - name: webinar-checker
-          image: gcr.forust.xyz/forust/webinar-checker:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/webinar-checker:prod
           ports:
             - name: metrics
               containerPort: 8000
               protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: metrics
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 12
+            initialDelaySeconds: 10
           envFrom:
             - secretRef:
                 name: edu-master-secrets
@@ -60,7 +67,7 @@ spec:
           resources:
             requests:
               cpu: "50m"
-              memory: "128Mi"
+              memory: "192Mi"
             limits:
               cpu: "600m"
-              memory: "512Mi"
+              memory: "384Mi"

errorpages/compose.yaml

@@ -1,7 +1,7 @@
 services:
   errorpage:
     build: .
-    image: gcr.forust.xyz/forust/error-pages:latest
+    image: gcr.forust.xyz/forust/error-pages:prod
     pull_policy: build
     container_name: error-pages
     restart: unless-stopped

errorpages/k8s/error-pages.yaml

@@ -27,7 +27,21 @@ spec:
     spec:
       containers:
         - name: error-pages
-          image: gcr.forust.xyz/forust/error-pages:latest
+          image: gcr.forust.xyz/forust/error-pages:prod
+          # p95 6M, max 10M, no limit before.
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "32Mi"
+            limits:
+              memory: "128Mi"
           ports:
             - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /404.html
+              port: 80
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
 ---

gitea/k8s/config.yaml

@@ -17,6 +17,12 @@ data:
 
   GITEA__mailer__ENABLED: "false"
 
+  # No code/issue search needed: bleve reindexes the whole issue index on
+  # every pod restart (cron.rebuild_issue_indexer RUN_AT_START) and hammers
+  # the rotational disk for an hour. "db" serves issue search from postgres.
+  GITEA__indexer__ISSUE_INDEXER_TYPE: "db"
+  GITEA__indexer__REPO_INDEXER_ENABLED: "false"
+
   GITEA__log__logger.access.MODE: "console, file"
   USER_UID: "1000"
   USER_GID: "1000"

gitea/k8s/gitea.yaml

@@ -47,10 +47,10 @@ spec:
               mountPath: /data
           resources:
             requests:
-              memory: "512Mi"
+              memory: "320Mi"
               cpu: "300m"
             limits:
-              memory: "1.5Gi"
+              memory: "1Gi"
               cpu: "1300m"
       volumes:
         - name: gitea-data

gitea/k8s/ingress.yaml

@@ -15,11 +15,15 @@ spec:
       services:
         - name: gitea-service
           port: 3000
+    # Registry route: NO crowdsec-bouncer.
+    # A deploy burst (runner Action API polls, `docker manifest inspect` per
+    # own image, containerd pulls, smoke probes) fires hundreds of parallel
+    # registry calls, and a ban on the runner breaks every later job. This
+    # route only serves authenticated OCI traffic - registry tokens and
+    # basic-auth are already handled by gitea - and scanners get nothing
+    # useful from /v2, so there is no bruteforce surface to protect here.
     - match: Host(`gcr.forust.xyz`) && PathPrefix(`/v2`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: gitea-service
           port: 3000

glance/k8s/glance.yaml

@@ -57,10 +57,10 @@ spec:
           resources:
             requests:
               cpu: "50m"
-              memory: "64Mi"
+              memory: "32Mi"
             limits:
               cpu: "200m"
-              memory: "256Mi"
+              memory: "128Mi"
       volumes:
         - name: glance-config
           configMap:

headscale/compose.yaml

@@ -1,6 +1,6 @@
 services:
   headscale:
-    image: headscale/headscale:0.29.3
+    image: headscale/headscale:v0.29.4
     restart: unless-stopped
     container_name: headscale-server
     command: serve

homepages/compose.yaml

@@ -3,7 +3,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.forust
-    image: gcr.forust.xyz/forust/forust-homepage:latest
+    image: gcr.forust.xyz/forust/forust-homepage:prod
     pull_policy: build
     # ports:
     #   - "8085:80"
@@ -35,7 +35,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.xdfnx
-    image: gcr.forust.xyz/forust/xdfnx-homepage:latest
+    image: gcr.forust.xyz/forust/xdfnx-homepage:prod
     pull_policy: build
     restart: unless-stopped
     # ports:

homepages/k8s/homepages.yaml

@@ -27,16 +27,22 @@ spec:
     spec:
       containers:
         - name: forust-homepage
-          image: gcr.forust.xyz/forust/forust-homepage:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/forust-homepage:prod
           ports:
             - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 80
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
           resources:
             requests:
-              memory: "10Mi"
+              memory: "32Mi"
               cpu: "20m"
             limits:
-              memory: "100Mi"
+              memory: "128Mi"
               cpu: "50m"
 ---
 apiVersion: v1
@@ -68,14 +74,20 @@ spec:
     spec:
       containers:
         - name: xdfnx-homepage
-          image: gcr.forust.xyz/forust/xdfnx-homepage:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/xdfnx-homepage:prod
           ports:
             - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 80
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
           resources:
             requests:
-              memory: "10Mi"
+              memory: "32Mi"
               cpu: "20m"
             limits:
-              memory: "100Mi"
+              memory: "128Mi"
               cpu: "50m"

immich/.env.example

@@ -0,0 +1,24 @@
+# You can find documentation for all the supported env variables at https://docs.immich.app/install/environment-variables
+
+# The location where your uploaded files are stored. The k8s manifests bind
+# mount /mnt/immich/library, which is the sdc9 partition - the same place, so
+# the two deployment paths look at one library.
+UPLOAD_LOCATION=/mnt/immich/library
+
+# The location where your database files are stored. Network shares are not supported for the database
+DB_DATA_LOCATION=./postgres
+
+# To set a timezone, uncomment the next line and change Etc/UTC to a TZ identifier from this list: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List
+# TZ=Etc/UTC
+
+# The Immich version to use. You can pin this to a specific version like "v2.1.0"
+IMMICH_VERSION=v3
+
+# Connection secret for postgres. You should change it to a random password
+# Please use only the characters `A-Za-z0-9`, without special characters or spaces
+DB_PASSWORD=postgres
+
+# The values below this line do not need to be changed
+###################################################################################
+DB_USERNAME=postgres
+DB_DATABASE_NAME=immich

immich/compose.yaml

@@ -0,0 +1,63 @@
+name: immich
+
+services:
+  immich-server:
+    container_name: immich_server
+    image: ghcr.io/immich-app/immich-server:v3
+    volumes:
+      - ${UPLOAD_LOCATION}:/data
+      - /etc/localtime:/etc/localtime:ro
+    env_file:
+      - .env
+    ports:
+      - "2283:2283"
+    depends_on:
+      - redis
+      - database
+    restart: always
+    healthcheck:
+      disable: false
+
+  immich-machine-learning:
+    container_name: immich_machine_learning
+    # For hardware acceleration, add one of -[armnn, cuda, rocm, openvino, rknn] to the image tag.
+    # Example tag: ${IMMICH_VERSION:-release}-cuda
+    image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}
+    # extends: # uncomment this section for hardware acceleration - see https://docs.immich.app/features/ml-hardware-acceleration
+    #   file: hwaccel.ml.yml
+    #   service: cpu # set to one of [armnn, cuda, rocm, openvino, openvino-wsl, rknn] for accelerated inference - use the `-wsl` version for WSL2 where applicable
+    volumes:
+      - model-cache:/cache
+    env_file:
+      - .env
+    restart: always
+    healthcheck:
+      disable: false
+
+  redis:
+    container_name: immich_redis
+    image: docker.io/valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd
+    healthcheck:
+      test: redis-cli ping | grep -q PONG || exit 1
+    restart: always
+
+  database:
+    container_name: immich_postgres
+    image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23
+    environment:
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_USER: ${DB_USERNAME}
+      POSTGRES_DB: ${DB_DATABASE_NAME}
+      POSTGRES_INITDB_ARGS: "--data-checksums"
+    # Uncomment the DB_STORAGE_TYPE: 'HDD' var if your database isn't stored on SSDs
+    # DB_STORAGE_TYPE: 'HDD'
+    volumes:
+      # Do not edit the next line. If you want to change the database storage location on your system, edit the value of DB_DATA_LOCATION in the .env file
+      - ${DB_DATA_LOCATION}:/var/lib/postgresql/data
+    shm_size: 128mb
+    restart: always
+    healthcheck:
+      disable: false
+
+volumes:
+  model-cache:

immich/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: immich-prod-tls
+  namespace: immich
+spec:
+  secretName: immich-prod-tls
+  dnsNames:
+    - immich.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: immich
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

immich/k8s/config.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: immich-config
+  namespace: immich
+data:
+  TZ: "Europe/Bratislava"
+
+  # The database in this namespace, not the shared one in the database
+  # namespace: v3 needs VectorChord, and only the dedicated image carries it.
+  DB_HOSTNAME: "immich-postgres"
+  DB_PORT: "5432"
+  DB_USERNAME: "immich"
+  DB_DATABASE_NAME: "immich"
+  DB_SSL_MODE: "disable"
+  DB_VECTOR_EXTENSION: "vectorchord"
+
+  REDIS_HOSTNAME: "immich-valkey"
+  REDIS_PORT: "6379"
+
+  # Traefik is the only client of the server, and it is a pod: the address immich
+  # sees is inside the node's pod CIDR. Without this the server does not trust
+  # X-Forwarded-For and every request looks like it came from Traefik itself.
+  IMMICH_TRUSTED_PROXIES: "10.244.0.0/24"

immich/k8s/immich.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-service
+  namespace: immich
+spec:
+  selector:
+    app: immich
+  ports:
+    - name: http
+      port: 2283
+      targetPort: 2283
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-deployment
+  namespace: immich
+  labels:
+    app: immich
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: immich
+  template:
+    metadata:
+      labels:
+        app: immich
+    spec:
+      containers:
+        - name: immich
+          image: ghcr.io/immich-app/immich-server:v3
+          envFrom:
+            - configMapRef:
+                name: immich-config
+            - secretRef:
+                name: immich-secrets
+          ports:
+            - name: http
+              containerPort: 2283
+          volumeMounts:
+            - name: immich-data
+              mountPath: /data
+          # The first boot runs migrations and warms the transcoder, which can
+          # take minutes, so liveness has to wait on the startup probe.
+          startupProbe:
+            httpGet:
+              path: /api/server/ping
+              port: http
+            failureThreshold: 60
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /api/server/ping
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /api/server/ping
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          # Only the request is scheduled against, and the node is already
+          # oversubscribed (5.58 of 6 cores requested) while actually running
+          # at about 1.5. So the request states what this sits at while idle -
+          # tens of millicores - and the limit leaves room for the burst that
+          # matters: thumbnails, transcodes and metadata extraction.
+          #
+          # The limit used to be 2Gi, but the server OOMKilled on boot while
+          # chewing through a backlog of unprocessed assets (API +Workers in
+          # one container spike well past idle).
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "512Mi"
+            limits:
+              cpu: "1500m"
+              memory: "4Gi"
+      volumes:
+        - name: immich-data
+          # The library lives on the node's own disk, not in a PVC. A PVC here
+          # meant declaring a size up front for data that does not exist yet,
+          # on a provisioner that cannot grow it, and the only copy of the
+          # photos was one `kubectl delete namespace` away.
+          #
+          # Directory, not DirectoryOrCreate, on purpose: if sdc9 is not
+          # mounted, this must fail loudly instead of quietly writing the
+          # library onto the root filesystem.
+          hostPath:
+            path: /mnt/immich/library
+            type: Directory

immich/k8s/ingress.yaml

@@ -0,0 +1,36 @@
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: immich-prod
+  namespace: immich
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`immich.forust.xyz`)
+      kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
+      services:
+        - name: immich-service
+          port: 2283
+  tls:
+    secretName: immich-prod-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: immich-local
+  namespace: immich
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`immich.workstation.internal`) || Host(`immich.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: immich-service
+          port: 2283
+  tls:
+    secretName: internal-wildcard-tls

immich/k8s/machine-learning.yaml

@@ -0,0 +1,92 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-machine-learning
+  namespace: immich
+spec:
+  selector:
+    app: immich-machine-learning
+  ports:
+    - name: http
+      port: 3003
+      targetPort: 3003
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-machine-learning-deployment
+  namespace: immich
+  labels:
+    app: immich-machine-learning
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: immich-machine-learning
+  template:
+    metadata:
+      labels:
+        app: immich-machine-learning
+    spec:
+      containers:
+        - name: immich-machine-learning
+          image: ghcr.io/immich-app/immich-machine-learning:v3
+          envFrom:
+            - configMapRef:
+                name: immich-config
+            - secretRef:
+                name: immich-secrets
+          ports:
+            - name: http
+              containerPort: 3003
+          volumeMounts:
+            - name: model-cache
+              mountPath: /cache
+          # The first request pulls a model over the internet, so a cold start
+          # is slower than a container start.
+          startupProbe:
+            httpGet:
+              path: /ping
+              port: http
+            failureThreshold: 60
+            periodSeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          # Same reasoning as the server: the request covers the idle cost
+          # only, because the node has no spare cores to schedule against.
+          # Recognition is the burst - a busy import wants both cores.
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "1Gi"
+            limits:
+              cpu: "2000m"
+              memory: "3Gi"
+      volumes:
+        - name: model-cache
+          persistentVolumeClaim:
+            claimName: immich-model-cache-pvc
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-model-cache-pvc
+  namespace: immich
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi

immich/k8s/namespace.yaml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=kubernetes
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: immich

immich/k8s/postgres.yaml

@@ -0,0 +1,138 @@
+# Immich's own database, separate from the shared postgres in the database
+# namespace. It has to be separate: v3 checks the VectorChord version at startup
+# and refuses to boot without it, VectorChord needs its .so in
+# shared_preload_libraries, and that can only be read when postmaster starts.
+# So the shared instance would have to be rebuilt on a custom image carrying
+# vchord and restarted - for every consumer of it (authentik, gitea, netbox,
+# netronome, penpot, statuspage). Not worth it for one photo library.
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-postgres
+  namespace: immich
+  labels:
+    app: immich-postgres
+spec:
+  selector:
+    app: immich-postgres
+  ports:
+    - name: postgres
+      port: 5432
+      targetPort: postgres
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: immich-postgres
+  namespace: immich
+  labels:
+    app: immich-postgres
+spec:
+  serviceName: immich-postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app: immich-postgres
+  template:
+    metadata:
+      labels:
+        app: immich-postgres
+    spec:
+      containers:
+        - name: postgres
+          # v3.x expects vchord for its vector work and vectors (pgvecto.rs)
+          # for some index types. This image ships both and preloads them, plus
+          # its own shared_buffers and wal settings, through
+          # /etc/postgresql/postgresql.conf - which its entrypoint reaches via
+          # `postgres -c config_file=...` in the image CMD.
+          #
+          # So there is deliberately no `command:` here. Overriding it replaces
+          # that config_file, and it also loses the step where the entrypoint
+          # drops from root to the postgres user: postmaster then starts as
+          # root and refuses to run.
+          image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23
+          env:
+            - name: POSTGRES_USER
+              value: immich
+            - name: POSTGRES_DB
+              value: immich
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: immich-secrets
+                  key: DB_PASSWORD
+            # Only read when the data directory is empty, so the checksums are
+            # decided here and never again.
+            - name: POSTGRES_INITDB_ARGS
+              value: --data-checksums
+            # The postgres-data volume lives on sdc, which is rotational. The
+            # SSD template is the default; HDD only changes the planner costs
+            # (effective_io_concurrency, random_page_cost), nothing structural.
+            - name: DB_STORAGE_TYPE
+              value: HDD
+            - name: TZ
+              valueFrom:
+                configMapKeyRef:
+                  name: immich-config
+                  key: TZ
+          ports:
+            - name: postgres
+              containerPort: 5432
+          volumeMounts:
+            - name: postgres-data
+              mountPath: /var/lib/postgresql/data
+            # The upstream compose file asks docker for 128mb of shm. Kubernetes
+            # gives every container 64mb, which is not what postmaster expects
+            # for parallel query workers and the WAL writer.
+            - name: shm
+              mountPath: /dev/shm
+          # Probes use a generous timeout on purpose: the data lives on a
+          # rotational disk on a loaded single node, and pg_isready can take
+          # seconds during WAL recovery. A 1s timeout kills the container
+          # mid-recovery and restarts the spiral.
+          startupProbe:
+            exec:
+              command: ["sh", "-c", "pg_isready -U immich -d immich"]
+            failureThreshold: 60
+            periodSeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", "pg_isready -U immich -d immich"]
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            exec:
+              command: ["sh", "-c", "pg_isready -U immich -d immich"]
+            initialDelaySeconds: 30
+            periodSeconds: 20
+            timeoutSeconds: 5
+          # The image template sets shared_buffers to 512MB, and the vchord and
+          # vectors workers are Rust binaries with a real RSS footprint on top
+          # of postmaster, checkpointer and friends. 1Gi was enough to start
+          # the server but the vectors worker kept dying in it, so the limit
+          # sits at 2Gi. The request stays at the idle cost.
+          resources:
+            requests:
+              cpu: "50m"
+              memory: "256Mi"
+            limits:
+              cpu: "1000m"
+              memory: "2Gi"
+      volumes:
+        - name: shm
+          emptyDir:
+            medium: Memory
+            sizeLimit: 128Mi
+  volumeClaimTemplates:
+    - metadata:
+        name: postgres-data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        # Retain: this is the metadata for a library that only exists in one
+        # place, and local-path cannot expand a bound volume, so this size has
+        # to hold until the library is rebuilt or dumped elsewhere.
+        storageClassName: local-path-retain
+        resources:
+          requests:
+            storage: 32Gi

immich/k8s/secrets.yaml.example

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: immich-secrets
+  namespace: immich
+type: Opaque
+stringData:
+  # Creates the immich superuser in this namespace's own postgres on first
+  # boot, and is the same value the server connects with. Nothing outside the
+  # immich namespace needs it. Letters and digits only: immich reads this into
+  # a connection string.
+  DB_PASSWORD: "changeme"
+  REDIS_PASSWORD: "changeme"

immich/k8s/valkey.yaml

@@ -0,0 +1,84 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-valkey
+  namespace: immich
+  labels:
+    app: immich-valkey
+spec:
+  clusterIP: None
+  selector:
+    app: immich-valkey
+  ports:
+    - name: valkey
+      port: 6379
+      targetPort: valkey
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: immich-valkey
+  namespace: immich
+  labels:
+    app: immich-valkey
+spec:
+  serviceName: immich-valkey
+  replicas: 1
+  selector:
+    matchLabels:
+      app: immich-valkey
+  template:
+    metadata:
+      labels:
+        app: immich-valkey
+    spec:
+      containers:
+        - name: valkey
+          image: docker.io/valkey/valkey:9.1.2-alpine
+          command:
+            - sh
+            - -c
+            - valkey-server --appendonly yes --save 30 1 --loglevel warning --requirepass "$REDIS_PASSWORD"
+          envFrom:
+            - secretRef:
+                name: immich-secrets
+          ports:
+            - name: valkey
+              containerPort: 6379
+          volumeMounts:
+            - name: valkey-data
+              mountPath: /data
+          # Same reasoning as postgres: 1s probe timeouts flap on a loaded
+          # single node with rotational storage.
+          startupProbe:
+            exec:
+              command: ["sh", "-c", 'valkey-cli --pass "$REDIS_PASSWORD" ping | grep -q PONG']
+            failureThreshold: 20
+            periodSeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", 'valkey-cli --pass "$REDIS_PASSWORD" ping | grep -q PONG']
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            exec:
+              command: ["sh", "-c", 'valkey-cli --pass "$REDIS_PASSWORD" ping | grep -q PONG']
+            initialDelaySeconds: 20
+            periodSeconds: 20
+            timeoutSeconds: 5
+          resources:
+            requests:
+              cpu: "25m"
+              memory: "64Mi"
+            limits:
+              cpu: "250m"
+              memory: "256Mi"
+  volumeClaimTemplates:
+    - metadata:
+        name: valkey-data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi

kener/compose.yaml

@@ -36,7 +36,7 @@ services:
       - proxy
       - kener
   redis:
-    image: redis:8.10.1-alpine
+    image: redis:8.10.2-alpine
     container_name: kener-redis
     restart: unless-stopped
     volumes:

kener/k8s/redis.yaml

@@ -30,7 +30,7 @@ spec:
     spec:
       containers:
         - name: redis
-          image: redis:8.10.1-alpine
+          image: redis:8.10.2-alpine
           ports:
             - containerPort: 6379
           volumeMounts:

loki/k8s/alloy-values.yaml

@@ -7,18 +7,32 @@
 
 controller:
   type: daemonset
+
+# config-reloader sidecar: p95 33M, max 43M. The chart keeps it at the top level,
+# not under `alloy:`.
+configReloader:
   resources:
     requests:
-      memory: "128Mi"
-      cpu: "50m"
+      memory: "32Mi"
+      cpu: "10m"
     limits:
-      memory: "512Mi"
-      cpu: "500m"
+      memory: "128Mi"
 
 image:
   tag: "v1.19.2"
 
 alloy:
+  # p95 275M, max 287M. Alloy tails every pod log and ships it to Loki, so it sits
+  # on the same IronWolf read path the node is I/O bound on. Request is set at p95.
+  # The chart key is `alloy.resources`. `controller.resources` is ignored silently,
+  # which is why this pod shipped with no limits at all.
+  resources:
+    requests:
+      memory: "288Mi"
+      cpu: "50m"
+    limits:
+      memory: "512Mi"
+
   configMap:
     create: true
     content: |

loki/k8s/loki-values.yaml

@@ -39,6 +39,16 @@ loki:
       local:
         directory: /var/loki/rules
 
+# p95 84M, max 85M for the rules sidecar that shares the singleBinary pod.
+# The chart exposes it as `sidecar.resources`, shared with any other sidecar.
+sidecar:
+  resources:
+    requests:
+      memory: "96Mi"
+      cpu: "10m"
+    limits:
+      memory: "192Mi"
+
 singleBinary:
   replicas: 1
   persistence:
@@ -47,10 +57,10 @@ singleBinary:
     storageClass: local-path-retain
   resources:
     requests:
-      memory: "512Mi"
+      memory: "256Mi"
       cpu: "200m"
     limits:
-      memory: "2Gi"
+      memory: "1Gi"
       cpu: "1000m"
 
 # Zeroed: unused in SingleBinary mode (chart validation requires it).
@@ -63,12 +73,25 @@ backend:
 
 gateway:
   replicas: 1
+  # Single node: chart default is required podAntiAffinity on hostname +
+  # RollingUpdate 25%/25% (effective maxUnavailable=0 at replicas=1).
+  # That deadlocks the rollout: the new pod stays Unschedulable while the
+  # old one lives, and the old one never leaves while the new one is not
+  # Ready. Null clears the default (an empty map would deep-merge with it
+  # and keep the required rule); maxUnavailable=1 allows a brief gateway
+  # outage during rollouts instead of a stuck deploy.
+  affinity: null
+  deploymentStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
   resources:
     requests:
-      memory: "64Mi"
+      memory: "32Mi"
       cpu: "50m"
     limits:
-      memory: "256Mi"
+      memory: "128Mi"
       cpu: "300m"
 
 monitoring:

metube/compose.yaml

@@ -1,6 +1,6 @@
 services:
   metube:
-    image: ghcr.io/alexta69/metube:2026.09.15
+    image: ghcr.io/alexta69/metube:2026.09.27
     container_name: metube
     restart: unless-stopped
     # ports:

metube/k8s/metube.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: metube
-          image: ghcr.io/alexta69/metube:2026.09.15
+          image: ghcr.io/alexta69/metube:2026.09.27
           envFrom:
             - configMapRef:
                 name: metube-config
@@ -36,12 +36,13 @@ spec:
           volumeMounts:
             - name: downloads
               mountPath: /downloads
+          # p95 72M, max 80M over 7 days. Was 600Mi/2Gi.
           resources:
             requests:
-              memory: "600Mi"
+              memory: "96Mi"
               cpu: "400m"
             limits:
-              memory: "2Gi"
+              memory: "384Mi"
               cpu: "1700m"
       volumes:
         - name: downloads

n8n/compose.yaml

@@ -1,6 +1,6 @@
 services:
   n8n:
-    image: docker.n8n.io/n8nio/n8n:2.41.0
+    image: docker.n8n.io/n8nio/n8n:2.41.3
     container_name: n8n
     restart: unless-stopped
     environment:

n8n/k8s/n8n.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: n8n
-          image: docker.n8n.io/n8nio/n8n:2.41.0
+          image: docker.n8n.io/n8nio/n8n:2.41.3
           envFrom:
             - configMapRef:
                 name: n8n-config

netbird/compose.yaml

@@ -87,7 +87,7 @@ services:
       - proxy
 
   dashboard:
-    image: netbirdio/dashboard:v2.90.10
+    image: netbirdio/dashboard:v2.93.0
     container_name: netbird-dashboard
     restart: unless-stopped
     environment:

netbird/k8s/ingress.yaml

@@ -7,12 +7,18 @@ spec:
   entryPoints:
     - websecure
   routes:
+    # NO crowdsec-bouncer on the API routes. These are the mesh client's own
+    # endpoints: gRPC-gateway management calls plus signal/relay long-polling,
+    # authenticated by NetBird's token rather than by a login form. A ban here
+    # is self-defeating - the client needs the mesh to reach anything else, so
+    # CrowdSec banning it locks the peer out of the network it needs to
+    # function. It also backfires: a banned peer keeps retrying, every retry
+    # is another 403, and LePresidente/http-generic-403-bf turns five 403s in
+    # ten seconds into a 4h ban, so one 403 loop kept re-arming the ban.
+    # netbird-local below has always been exempt; this makes prod match.
     - match: Host(`nb.forust.xyz`) && (PathPrefix(`/signalexchange.SignalExchange/`) || PathPrefix(`/management.ManagementService/`) || PathPrefix(`/management.ProxyService/`))
       kind: Rule
       priority: 100
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: netbird-server-service
           port: 80
@@ -20,9 +26,6 @@ spec:
     - match: Host(`nb.forust.xyz`) && (PathPrefix(`/relay`) || PathPrefix(`/ws-proxy/`) || PathPrefix(`/api`) || PathPrefix(`/oauth2`))
       kind: Rule
       priority: 100
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: netbird-server-service
           port: 80

netbird/k8s/netbird.yaml

@@ -88,12 +88,13 @@ spec:
             periodSeconds: 30
             timeoutSeconds: 5
             failureThreshold: 5
+          # p95 97M, max 102M over 7 days. Was 256Mi/1Gi.
           resources:
             requests:
-              memory: "256Mi"
+              memory: "128Mi"
               cpu: "250m"
             limits:
-              memory: "1Gi"
+              memory: "384Mi"
               cpu: "1000m"
       volumes:
         - name: netbird-data
@@ -137,7 +138,7 @@ spec:
     spec:
       containers:
         - name: dashboard
-          image: netbirdio/dashboard:v2.90.10
+          image: netbirdio/dashboard:v2.93.0
           envFrom:
             - configMapRef:
                 name: netbird-config
@@ -162,10 +163,10 @@ spec:
             failureThreshold: 5
           resources:
             requests:
-              memory: "64Mi"
+              memory: "32Mi"
               cpu: "50m"
             limits:
-              memory: "256Mi"
+              memory: "128Mi"
               cpu: "300m"
 ---
 apiVersion: v1

netbox/.env.example

@@ -0,0 +1,32 @@
+POSTGRES_DB=netbox
+POSTGRES_USER=netbox
+POSTGRES_PASSWORD=CHANGE_ME_POSTGRES_PASSWORD
+
+DB_NAME=netbox
+DB_USER=netbox
+DB_PASSWORD=CHANGE_ME_POSTGRES_PASSWORD
+DB_HOST=postgres
+DB_PORT=5432
+DB_SSLMODE=disable
+
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_PASSWORD=CHANGE_ME_REDIS_PASSWORD
+REDIS_DATABASE=0
+REDIS_CACHE_HOST=redis-cache
+REDIS_CACHE_PORT=6379
+REDIS_CACHE_PASSWORD=CHANGE_ME_REDIS_CACHE_PASSWORD
+REDIS_CACHE_DATABASE=1
+
+ALLOWED_HOSTS=localhost,127.0.0.1,[::1],netbox.forust.xyz,netbox.workstation.internal
+CSRF_TRUSTED_ORIGINS=https://netbox.forust.xyz,https://netbox.workstation.internal
+
+SECRET_KEY=CHANGE_ME_DJANGO_SECRET_KEY
+API_TOKEN_PEPPER_1=CHANGE_ME_API_TOKEN_PEPPER
+TIME_ZONE=Europe/Bratislava
+TZ=Europe/Bratislava
+
+SKIP_SUPERUSER=false
+SUPERUSER_NAME=admin
+SUPERUSER_EMAIL=admin@example.com
+SUPERUSER_PASSWORD=CHANGE_ME_SUPERUSER_PASSWORD

netbox/README.md

@@ -0,0 +1,96 @@
+# NetBox
+
+NetBox for homelab documentation and visualization. Two runtimes are available:
+
+| Runtime | Manifest       | Purpose                                                        |
+| ------- | -------------- | -------------------------------------------------------------- |
+| Docker  | `compose.yaml` | Local stand on `127.0.0.1:8000` (no public exposure)           |
+| k8s     | `k8s/`         | Homelab service on `netbox.forust.xyz` (and the internal name) |
+
+Both use the same image (`netboxcommunity/netbox:v4.7-5.1.1`) and Valkey for tasks
+plus a second logical database for caching. The Docker stand keeps its own
+PostgreSQL container, while the k8s deployment uses the shared `database` cluster
+(`postgres.database.svc.cluster.local:5432`, role/database `netbox`); only Valkey
+stays a per-service StatefulSet.
+
+## Docker Compose
+
+```bash
+cp .env.example .env
+# replace CHANGE_ME
+docker compose up -d
+```
+
+The UI is available at <http://localhost:8000>. The port is bound to `127.0.0.1`
+intentionally, so this stand is not exposed on the LAN or public interfaces.
+
+The `netbox` service is also attached to the external `proxy` network and carries
+Traefik labels for `netbox.forust.xyz` and `netbox.workstation.internal`. Those
+labels only take effect while the Docker Traefik stack is running; it is currently
+stopped, and the live ingress path in this homelab is the k8s Traefik.
+
+Inspect startup and health with:
+
+```bash
+docker compose ps
+docker compose logs -f netbox
+```
+
+Stop it with `docker compose down`; data is kept in the named volumes
+`netbox-postgres`, `netbox-media-files`, `netbox-reports-files`,
+`netbox-scripts-files` and `netbox-redis-data`.
+
+## Kubernetes
+
+`k8s/` is deployed in the homelab cluster and serves `netbox.forust.xyz` publicly
+plus `netbox.workstation.internal` / `netbox.gigaforust.internal` internally. To
+rebuild it from scratch:
+
+```bash
+# 1. shared PostgreSQL: the password lives in the shared secret, NetBox keeps a copy
+kubectl -n database patch secret postgres-shared-secrets \
+  --type merge -p '{"stringData":{"NETBOX_DB_PASSWORD":"<same value>"}}'
+kubectl -n database exec postgres17-0 -- psql -U postgres -d postgres \
+  -c 'CREATE ROLE netbox LOGIN PASSWORD ...' -c 'CREATE DATABASE netbox OWNER netbox'
+
+# 2. secrets first: the deploy workflow never applies *secret*.yaml
+cp k8s/secrets.yaml.example k8s/secrets.yaml   # replace CHANGE_ME
+kubectl apply -f k8s/secrets.yaml
+
+# 3. manifests
+kubectl apply -f k8s/
+```
+
+The shared cluster is reached at `postgres.database.svc.cluster.local:5432`. Its
+NetworkPolicy (`postgres/k8s/network-policy.yaml`) must list the `netbox` namespace
+or connections are dropped, and `postgres/initdb/01-create-databases.sh` already
+creates the role and database on a fresh data directory. NetBox has no PostgreSQL
+StatefulSet of its own — only `netbox-valkey`.
+
+`netbox.forust.xyz` resolves to this host (`78.98.72.122`) through the `DOMAINS`
+list in the `default/cfddns` secret. cert-manager issues `netbox-prod-tls` with the
+`letsencrypt-prod` issuer, the internal route uses `internal-wildcard-tls`.
+
+Resources are permanent again now that the first-boot migrations are complete:
+the web container reserves `100m`/`512Mi` and is capped at `2` CPU/`2Gi`, the
+worker reserves `50m`/`256Mi` and is capped at `1` CPU/`1Gi`, and Valkey reserves
+`25m`/`64Mi` and is capped at `250m`/`256Mi`. The deliberately generous CPU caps
+leave enough headroom for future schema migrations without letting one process
+consume the whole node.
+
+The first start applies ~810 migrations, each in its own transaction with DDL and
+a commit; every later start is a no-op. The startup probe allows 15 minutes and
+`progressDeadlineSeconds` is 1800 for the same reason. Probes run inside the pod
+and explicitly set `Host: netbox.forust.xyz`; a kubelet `httpGet.host` field would
+replace the probe destination with that public hostname and bypass the pod.
+
+## Secrets
+
+- `netbox/.env` (compose) and `netbox/k8s/secrets.yaml` (k8s) are gitignored. Only
+  `.env.example` and `k8s/secrets.yaml.example` are committed.
+- `netbox/configuration/configuration.py` is env-driven: hosts, database, Redis and
+  the Django keys all come from the environment, so the same settings file works in
+  both runtimes. The k8s copy lives in the `netbox-settings` ConfigMap
+  (`k8s/settings.yaml`) and must be kept in sync with the file.
+- Rotating `SECRET_KEY` invalidates all sessions; rotating `API_TOKEN_PEPPER_1`
+  invalidates every API token.

netbox/compose.yaml

@@ -0,0 +1,137 @@
+services:
+  netbox:
+    image: docker.io/netboxcommunity/netbox:v4.7-5.1.1
+    container_name: netbox
+    restart: unless-stopped
+    user: "netbox:root"
+    ports:
+      - "127.0.0.1:8000:8080"
+    env_file:
+      - .env
+    environment:
+      GRANIAN_WORKERS: "2"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      redis-cache:
+        condition: service_healthy
+    volumes:
+      - ./configuration:/etc/netbox/config:z,ro
+      - netbox-media-files:/opt/netbox/netbox/media
+      - netbox-reports-files:/opt/netbox/netbox/reports
+      - netbox-scripts-files:/opt/netbox/netbox/scripts
+    networks:
+      - default
+      - proxy
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.netbox.loadbalancer.server.port=8080"
+
+      # Prod Router
+      - "traefik.http.routers.netbox.rule=Host(`netbox.forust.xyz`)"
+      - "traefik.http.routers.netbox.entrypoints=websecure"
+      - "traefik.http.routers.netbox.middlewares=security-headers@file"
+      - "traefik.http.routers.netbox.tls.certresolver=letsencrypt"
+      # Local Router
+      - "traefik.http.routers.netbox-local.rule=Host(`netbox.workstation.internal`)"
+      - "traefik.http.routers.netbox-local.entrypoints=websecure"
+      - "traefik.http.routers.netbox-local.tls=true"
+    healthcheck:
+      test: ["CMD", "/opt/netbox/health.sh"]
+      start_period: 600s
+      timeout: 5s
+      interval: 15s
+      retries: 10
+
+  netbox-worker:
+    image: docker.io/netboxcommunity/netbox:v4.7-5.1.1
+    container_name: netbox-worker
+    restart: unless-stopped
+    user: "netbox:root"
+    command:
+      - /opt/netbox/venv/bin/python
+      - /opt/netbox/netbox/manage.py
+      - rqworker
+    env_file:
+      - .env
+    depends_on:
+      netbox:
+        condition: service_healthy
+    volumes:
+      - ./configuration:/etc/netbox/config:z,ro
+      - netbox-media-files:/opt/netbox/netbox/media
+      - netbox-reports-files:/opt/netbox/netbox/reports
+      - netbox-scripts-files:/opt/netbox/netbox/scripts
+    healthcheck:
+      test: ["CMD-SHELL", "ps -ef | grep -q '[r]qworker'"]
+      start_period: 30s
+      timeout: 5s
+      interval: 15s
+      retries: 10
+
+  postgres:
+    image: docker.io/postgres:18.6-alpine
+    container_name: netbox-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: "${POSTGRES_DB:?POSTGRES_DB must be set}"
+      POSTGRES_USER: "${POSTGRES_USER:?POSTGRES_USER must be set}"
+      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set}"
+    volumes:
+      - netbox-postgres:/var/lib/postgresql
+    healthcheck:
+      test: ["CMD-SHELL", 'pg_isready -q -t 2 -d "$${POSTGRES_DB}" -U "$${POSTGRES_USER}"']
+      start_period: 20s
+      timeout: 5s
+      interval: 10s
+      retries: 10
+
+  redis:
+    image: docker.io/valkey/valkey:9.1.2-alpine
+    container_name: netbox-redis
+    restart: unless-stopped
+    command:
+      - sh
+      - -c
+      - valkey-server --appendonly yes --requirepass "$$REDIS_PASSWORD"
+    environment:
+      REDIS_PASSWORD: "${REDIS_PASSWORD:?REDIS_PASSWORD must be set}"
+    volumes:
+      - netbox-redis-data:/data
+    healthcheck:
+      test: ["CMD-SHELL", 'valkey-cli --pass "$${REDIS_PASSWORD}" ping | grep -q PONG']
+      start_period: 5s
+      timeout: 5s
+      interval: 5s
+      retries: 10
+
+  redis-cache:
+    image: docker.io/valkey/valkey:9.1.2-alpine
+    container_name: netbox-redis-cache
+    restart: unless-stopped
+    command:
+      - sh
+      - -c
+      - valkey-server --requirepass "$$REDIS_CACHE_PASSWORD"
+    environment:
+      REDIS_CACHE_PASSWORD: "${REDIS_CACHE_PASSWORD:?REDIS_CACHE_PASSWORD must be set}"
+    healthcheck:
+      test: ["CMD-SHELL", 'valkey-cli --pass "$${REDIS_CACHE_PASSWORD}" ping | grep -q PONG']
+      start_period: 5s
+      timeout: 5s
+      interval: 5s
+      retries: 10
+
+volumes:
+  netbox-media-files:
+  netbox-reports-files:
+  netbox-scripts-files:
+  netbox-postgres:
+  netbox-redis-data:
+
+networks:
+  default:
+  proxy:
+    external: true

netbox/configuration/configuration.py

@@ -0,0 +1,48 @@
+import os
+
+
+def _csv(name, default=''):
+    return [item.strip() for item in os.environ.get(name, default).split(',') if item.strip()]
+
+
+ALLOWED_HOSTS = _csv('ALLOWED_HOSTS', 'localhost,127.0.0.1,[::1]')
+CSRF_TRUSTED_ORIGINS = _csv('CSRF_TRUSTED_ORIGINS')
+USE_X_FORWARDED_HOST = True
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+
+DATABASES = {
+    'default': {
+        'NAME': os.environ['DB_NAME'],
+        'USER': os.environ['DB_USER'],
+        'PASSWORD': os.environ['DB_PASSWORD'],
+        'HOST': os.environ['DB_HOST'],
+        'PORT': os.environ.get('DB_PORT', '5432'),
+        'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'disable')},
+        'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')),
+    }
+}
+
+REDIS = {
+    'tasks': {
+        'HOST': os.environ['REDIS_HOST'],
+        'PORT': int(os.environ.get('REDIS_PORT', '6379')),
+        'PASSWORD': os.environ['REDIS_PASSWORD'],
+        'DATABASE': int(os.environ.get('REDIS_DATABASE', '0')),
+        'SSL': False,
+    },
+    'caching': {
+        'HOST': os.environ['REDIS_CACHE_HOST'],
+        'PORT': int(os.environ.get('REDIS_CACHE_PORT', '6379')),
+        'PASSWORD': os.environ['REDIS_CACHE_PASSWORD'],
+        'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', '1')),
+        'SSL': False,
+    },
+}
+
+SECRET_KEY = os.environ['SECRET_KEY']
+API_TOKEN_PEPPERS = {1: os.environ['API_TOKEN_PEPPER_1']}
+TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC')
+MEDIA_ROOT = '/opt/netbox/netbox/media'
+REPORTS_ROOT = '/opt/netbox/netbox/reports'
+SCRIPTS_ROOT = '/opt/netbox/netbox/scripts'
+CENSUS_REPORTING_ENABLED = False

netbox/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: netbox-prod-tls
+  namespace: netbox
+spec:
+  secretName: netbox-prod-tls
+  dnsNames:
+    - netbox.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: netbox
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

netbox/k8s/config.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: netbox-config
+  namespace: netbox
+data:
+  DB_HOST: "postgres.database.svc.cluster.local"
+  DB_PORT: "5432"
+  DB_SSLMODE: "disable"
+  REDIS_HOST: "netbox-valkey"
+  REDIS_PORT: "6379"
+  REDIS_DATABASE: "0"
+  REDIS_CACHE_HOST: "netbox-valkey"
+  REDIS_CACHE_PORT: "6379"
+  REDIS_CACHE_DATABASE: "1"
+  TIME_ZONE: "Europe/Bratislava"
+  TZ: "Europe/Bratislava"
+  GRANIAN_WORKERS: "2"
+  ALLOWED_HOSTS: "netbox.forust.xyz,netbox.workstation.internal,netbox.gigaforust.internal"
+  CSRF_TRUSTED_ORIGINS: "https://netbox.forust.xyz,https://netbox.workstation.internal,https://netbox.gigaforust.internal"
+  SKIP_SUPERUSER: "false"

netbox/k8s/ingress.yaml

@@ -0,0 +1,36 @@
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: netbox-prod
+  namespace: netbox
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`netbox.forust.xyz`)
+      kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
+      services:
+        - name: netbox-service
+          port: 8080
+  tls:
+    secretName: netbox-prod-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: netbox-local
+  namespace: netbox
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`netbox.workstation.internal`) || Host(`netbox.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: netbox-service
+          port: 8080
+  tls:
+    secretName: internal-wildcard-tls

netbox/k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: netbox

netbox/k8s/netbox.yaml

@@ -0,0 +1,213 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: netbox-service
+  namespace: netbox
+spec:
+  selector:
+    app: netbox
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netbox-deployment
+  namespace: netbox
+  labels:
+    app: netbox
+spec:
+  replicas: 1
+  progressDeadlineSeconds: 300
+  selector:
+    matchLabels:
+      app: netbox
+  strategy:
+    # ReadWriteOnce PVC
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: netbox
+    spec:
+      containers:
+        - name: netbox
+          image: docker.io/netboxcommunity/netbox:v4.7-5.1.1
+          ports:
+            - name: http
+              containerPort: 8080
+          envFrom:
+            - configMapRef:
+                name: netbox-config
+            - secretRef:
+                name: netbox-secrets
+          volumeMounts:
+            - name: netbox-config
+              mountPath: /etc/netbox/config
+              readOnly: true
+            - name: netbox-media
+              mountPath: /opt/netbox/netbox/media
+            - name: netbox-reports
+              mountPath: /opt/netbox/netbox/reports
+            - name: netbox-scripts
+              mountPath: /opt/netbox/netbox/scripts
+          startupProbe:
+            exec:
+              command:
+                - /usr/bin/curl
+                - --fail
+                - --silent
+                - --show-error
+                - --max-time
+                - "4"
+                - --header
+                - "Host: netbox.forust.xyz"
+                - http://127.0.0.1:8080/login/
+            failureThreshold: 90
+            periodSeconds: 10
+          readinessProbe:
+            exec:
+              command:
+                - /usr/bin/curl
+                - --fail
+                - --silent
+                - --show-error
+                - --max-time
+                - "4"
+                - --header
+                - "Host: netbox.forust.xyz"
+                - http://127.0.0.1:8080/login/
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command:
+                - /usr/bin/curl
+                - --fail
+                - --silent
+                - --show-error
+                - --max-time
+                - "4"
+                - --header
+                - "Host: netbox.forust.xyz"
+                - http://127.0.0.1:8080/login/
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "1Gi"
+            limits:
+              cpu: "2"
+              memory: "2Gi"
+      volumes:
+        - name: netbox-config
+          configMap:
+            name: netbox-settings
+        - name: netbox-media
+          persistentVolumeClaim:
+            claimName: netbox-media-pvc
+        - name: netbox-reports
+          persistentVolumeClaim:
+            claimName: netbox-reports-pvc
+        - name: netbox-scripts
+          persistentVolumeClaim:
+            claimName: netbox-scripts-pvc
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netbox-worker-deployment
+  namespace: netbox
+  labels:
+    app: netbox-worker
+spec:
+  replicas: 1
+  progressDeadlineSeconds: 300
+  selector:
+    matchLabels:
+      app: netbox-worker
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: netbox-worker
+    spec:
+      containers:
+        - name: netbox-worker
+          image: docker.io/netboxcommunity/netbox:v4.7-5.1.1
+          command:
+            - /opt/netbox/venv/bin/python
+            - netbox/manage.py
+            - rqworker
+          workingDir: /opt/netbox
+          envFrom:
+            - configMapRef:
+                name: netbox-config
+            - secretRef:
+                name: netbox-secrets
+          volumeMounts:
+            - name: netbox-config
+              mountPath: /etc/netbox/config
+              readOnly: true
+            - name: netbox-media
+              mountPath: /opt/netbox/netbox/media
+            - name: netbox-reports
+              mountPath: /opt/netbox/netbox/reports
+            - name: netbox-scripts
+              mountPath: /opt/netbox/netbox/scripts
+          resources:
+            requests:
+              cpu: "50m"
+              memory: "256Mi"
+            limits:
+              cpu: "1"
+              memory: "512Mi"
+      volumes:
+        - name: netbox-config
+          configMap:
+            name: netbox-settings
+        - name: netbox-media
+          persistentVolumeClaim:
+            claimName: netbox-media-pvc
+        - name: netbox-reports
+          persistentVolumeClaim:
+            claimName: netbox-reports-pvc
+        - name: netbox-scripts
+          persistentVolumeClaim:
+            claimName: netbox-scripts-pvc
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: netbox-media-pvc
+  namespace: netbox
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: netbox-reports-pvc
+  namespace: netbox
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: netbox-scripts-pvc
+  namespace: netbox
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi

netbox/k8s/secrets.yaml.example

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: netbox-secrets
+  namespace: netbox
+type: Opaque
+stringData:
+  DB_NAME: "netbox"
+  DB_USER: "netbox"
+  DB_PASSWORD: "CHANGE_ME_POSTGRES_PASSWORD"
+  REDIS_PASSWORD: "CHANGE_ME_VALKEY_PASSWORD"
+  REDIS_CACHE_PASSWORD: "CHANGE_ME_VALKEY_PASSWORD"
+  VALKEY_PASSWORD: "CHANGE_ME_VALKEY_PASSWORD"
+  SECRET_KEY: "CHANGE_ME_DJANGO_SECRET_KEY"
+  API_TOKEN_PEPPER_1: "CHANGE_ME_API_TOKEN_PEPPER"
+  SUPERUSER_NAME: "admin"
+  SUPERUSER_EMAIL: "admin@example.com"
+  SUPERUSER_PASSWORD: "CHANGE_ME_SUPERUSER_PASSWORD"

netbox/k8s/settings.yaml

@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: netbox-settings
+  namespace: netbox
+data:
+  # Sync wit netbox/configuration/configuration.py (the Docker mounts that file).
+  configuration.py: |
+    import os
+
+
+    def _csv(name, default=""):
+        return [item.strip() for item in os.environ.get(name, default).split(",") if item.strip()]
+
+
+    ALLOWED_HOSTS = _csv("ALLOWED_HOSTS", "localhost,127.0.0.1,[::1]")
+    CSRF_TRUSTED_ORIGINS = _csv("CSRF_TRUSTED_ORIGINS")
+    USE_X_FORWARDED_HOST = True
+    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+    DATABASES = {
+        "default": {
+            "NAME": os.environ["DB_NAME"],
+            "USER": os.environ["DB_USER"],
+            "PASSWORD": os.environ["DB_PASSWORD"],
+            "HOST": os.environ["DB_HOST"],
+            "PORT": os.environ.get("DB_PORT", "5432"),
+            "OPTIONS": {"sslmode": os.environ.get("DB_SSLMODE", "disable")},
+            "CONN_MAX_AGE": int(os.environ.get("DB_CONN_MAX_AGE", "300")),
+        }
+    }
+
+    REDIS = {
+        "tasks": {
+            "HOST": os.environ["REDIS_HOST"],
+            "PORT": int(os.environ.get("REDIS_PORT", "6379")),
+            "PASSWORD": os.environ["REDIS_PASSWORD"],
+            "DATABASE": int(os.environ.get("REDIS_DATABASE", "0")),
+            "SSL": False,
+        },
+        "caching": {
+            "HOST": os.environ["REDIS_CACHE_HOST"],
+            "PORT": int(os.environ.get("REDIS_CACHE_PORT", "6379")),
+            "PASSWORD": os.environ["REDIS_CACHE_PASSWORD"],
+            "DATABASE": int(os.environ.get("REDIS_CACHE_DATABASE", "1")),
+            "SSL": False,
+        },
+    }
+
+    SECRET_KEY = os.environ["SECRET_KEY"]
+    API_TOKEN_PEPPERS = {1: os.environ["API_TOKEN_PEPPER_1"]}
+    TIME_ZONE = os.environ.get("TIME_ZONE", "UTC")
+    MEDIA_ROOT = "/opt/netbox/netbox/media"
+    REPORTS_ROOT = "/opt/netbox/netbox/reports"
+    SCRIPTS_ROOT = "/opt/netbox/netbox/scripts"
+    CENSUS_REPORTING_ENABLED = False

netbox/k8s/valkey.yaml

@@ -0,0 +1,82 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: netbox-valkey
+  namespace: netbox
+  labels:
+    app: netbox-valkey
+spec:
+  clusterIP: None
+  selector:
+    app: netbox-valkey
+  ports:
+    - name: valkey
+      port: 6379
+      targetPort: valkey
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: netbox-valkey
+  namespace: netbox
+  labels:
+    app: netbox-valkey
+spec:
+  serviceName: netbox-valkey
+  replicas: 1
+  selector:
+    matchLabels:
+      app: netbox-valkey
+  template:
+    metadata:
+      labels:
+        app: netbox-valkey
+    spec:
+      containers:
+        - name: valkey
+          image: docker.io/valkey/valkey:9.1.2-alpine
+          command:
+            - sh
+            - -c
+            - valkey-server --appendonly yes --save 30 1 --loglevel warning --requirepass "$VALKEY_PASSWORD"
+          env:
+            - name: VALKEY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: netbox-secrets
+                  key: VALKEY_PASSWORD
+          ports:
+            - name: valkey
+              containerPort: 6379
+          volumeMounts:
+            - name: valkey-data
+              mountPath: /data
+          startupProbe:
+            exec:
+              command: ["sh", "-c", 'valkey-cli --pass "$VALKEY_PASSWORD" ping | grep -q PONG']
+            failureThreshold: 20
+            periodSeconds: 5
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", 'valkey-cli --pass "$VALKEY_PASSWORD" ping | grep -q PONG']
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["sh", "-c", 'valkey-cli --pass "$VALKEY_PASSWORD" ping | grep -q PONG']
+            initialDelaySeconds: 20
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: "25m"
+              memory: "64Mi"
+            limits:
+              cpu: "250m"
+              memory: "256Mi"
+  volumeClaimTemplates:
+    - metadata:
+        name: valkey-data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi

netronome/compose.yaml

@@ -1,6 +1,6 @@
 services:
   netronome:
-    image: ghcr.io/autobrr/netronome:v0.14.0
+    image: ghcr.io/autobrr/netronome:v0.14.1
     restart: unless-stopped
     container_name: netronome
     ports:

netronome/k8s/netronome.yaml

@@ -30,7 +30,7 @@ spec:
     spec:
       containers:
         - name: netronome
-          image: ghcr.io/autobrr/netronome:v0.14.0
+          image: ghcr.io/autobrr/netronome:v0.14.1
           ports:
             - name: netronome-port
               protocol: TCP
@@ -51,8 +51,8 @@ spec:
                   key: NETRONOME__DB_PASSWORD
           resources:
             requests:
-              memory: "100Mi"
+              memory: "64Mi"
               cpu: "100m"
             limits:
-              memory: "512Mi"
+              memory: "256Mi"
               cpu: "500m"

postgres/README.md

@@ -1,7 +1,7 @@
 # Shared PostgreSQL
 
 This directory contains the shared PostgreSQL 17 deployment for Authentik,
-Gitea, Netronome, and Statuspage. It creates one database and one login role
+Gitea, NetBox, Netronome, and Statuspage. It creates one database and one login role
 per service. Per-service standalone databases were removed after the
 migration (Sep 2026); Penpot stays on its own compose PostgreSQL (archived,
 not part of the shared instance).
@@ -12,6 +12,7 @@ not part of the shared instance).
 | ---------- | ------------------- | -------------------------------------- |
 | Authentik  | 2025.10.x           | Supported (Authentik requires 14+)     |
 | Gitea      | 1.27.3              | Supported (Gitea requires 12+)         |
+| NetBox     | 4.7.x               | Supported (NetBox 4.x requires 13+)    |
 | Netronome  | 0.14.0              | Supported (upstream's example uses 17) |
 | Statuspage | custom              | Supported                              |
 

postgres/initdb/01-create-databases.sh

@@ -3,6 +3,7 @@ set -euo pipefail
 
 : "${AUTHENTIK_DB_PASSWORD:?AUTHENTIK_DB_PASSWORD is required}"
 : "${GITEA_DB_PASSWORD:?GITEA_DB_PASSWORD is required}"
+: "${NETBOX_DB_PASSWORD:?NETBOX_DB_PASSWORD is required}"
 : "${NETRONOME_DB_PASSWORD:?NETRONOME_DB_PASSWORD is required}"
 : "${PENPOT_DB_PASSWORD:?PENPOT_DB_PASSWORD is required}"
 : "${STATUSPAGE_DB_PASSWORD:?STATUSPAGE_DB_PASSWORD is required}"
@@ -23,6 +24,7 @@ SQL
 
 create_role_and_database authentik authentik "$AUTHENTIK_DB_PASSWORD"
 create_role_and_database gitea gitea "$GITEA_DB_PASSWORD"
+create_role_and_database netbox netbox "$NETBOX_DB_PASSWORD"
 create_role_and_database netronome netronome "$NETRONOME_DB_PASSWORD"
 create_role_and_database penpot penpot "$PENPOT_DB_PASSWORD"
 create_role_and_database statuspage statuspage "$STATUSPAGE_DB_PASSWORD"

postgres/k8s/network-policy.yaml

@@ -17,6 +17,9 @@ spec:
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: gitea
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: netbox
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: netronome

postgres/k8s/postgres.yaml

@@ -60,16 +60,26 @@ spec:
               command: ["pg_isready", "-U", "postgres", "-d", "postgres"]
             initialDelaySeconds: 10
             periodSeconds: 10
+            # Generous timeout: on an I/O-bound single node even exec can take
+            # seconds, and a 1s default kills a healthy postgres mid-recovery.
+            timeoutSeconds: 5
           startupProbe:
             exec:
               command: ["pg_isready", "-U", "postgres", "-d", "postgres"]
-            failureThreshold: 30
+            # Crash recovery on an I/O-starved single node can fsync for 10+
+            # minutes; killing postgres mid-recovery restarts the fsync from
+            # zero and loops forever. 90x10s = 15 minutes of grace.
+            failureThreshold: 90
             periodSeconds: 10
           livenessProbe:
             exec:
               command: ["pg_isready", "-U", "postgres", "-d", "postgres"]
             initialDelaySeconds: 30
             periodSeconds: 20
+            # Same I/O reasoning as readiness, plus more misses before a kill:
+            # restarting postgres on a loaded node only makes recovery longer.
+            timeoutSeconds: 5
+            failureThreshold: 5
           resources:
             requests:
               memory: "512Mi"
@@ -113,6 +123,7 @@ data:
 
     : "${AUTHENTIK_DB_PASSWORD:?AUTHENTIK_DB_PASSWORD is required}"
     : "${GITEA_DB_PASSWORD:?GITEA_DB_PASSWORD is required}"
+    : "${NETBOX_DB_PASSWORD:?NETBOX_DB_PASSWORD is required}"
     : "${NETRONOME_DB_PASSWORD:?NETRONOME_DB_PASSWORD is required}"
     : "${PENPOT_DB_PASSWORD:?PENPOT_DB_PASSWORD is required}"
     : "${STATUSPAGE_DB_PASSWORD:?STATUSPAGE_DB_PASSWORD is required}"
@@ -133,6 +144,7 @@ data:
 
     create_role_and_database authentik authentik "$AUTHENTIK_DB_PASSWORD"
     create_role_and_database gitea gitea "$GITEA_DB_PASSWORD"
+    create_role_and_database netbox netbox "$NETBOX_DB_PASSWORD"
     create_role_and_database netronome netronome "$NETRONOME_DB_PASSWORD"
     create_role_and_database penpot penpot "$PENPOT_DB_PASSWORD"
     create_role_and_database statuspage statuspage "$STATUSPAGE_DB_PASSWORD"

postgres/k8s/secrets.yaml.example

@@ -8,6 +8,7 @@ stringData:
   POSTGRES_ADMIN_PASSWORD: ""
   AUTHENTIK_DB_PASSWORD: ""
   GITEA_DB_PASSWORD: ""
+  NETBOX_DB_PASSWORD: ""
   NETRONOME_DB_PASSWORD: ""
   PENPOT_DB_PASSWORD: ""
   STATUSPAGE_DB_PASSWORD: ""

prometheus-stack/compose.yaml

@@ -30,7 +30,7 @@ services:
       - proxy
 
   prometheus:
-    image: prom/prometheus:v3.14.0
+    image: prom/prometheus:v3.15.0
     container_name: prometheus-prometheus
     restart: unless-stopped
     command:

prometheus-stack/k8s/alerts.yaml

@@ -27,6 +27,33 @@ spec:
             summary: "Pod is crash looping"
             description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff."
 
+        - alert: ContainerOOMKilled
+          expr: max_over_time(kube_pod_container_status_terminated_reason{reason="OOMKilled"}[15m]) >= 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Container was OOMKilled"
+            description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} was killed for exceeding its memory limit. Raise the limit or reduce the workload."
+
+        - alert: ContainerRestartingTooOften
+          expr: max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total[30m])) > 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Container restarting too often"
+            description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value }} times in the last 30 minutes."
+
+        - alert: PodEvicted
+          expr: max_over_time(kube_pod_status_reason{reason="Evicted"}[15m]) >= 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Pod was evicted"
+            description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} was evicted, usually for node disk or memory pressure."
+
         - alert: PersistentVolumeClaimFillingUp
           expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15
           for: 15m

prometheus-stack/k8s/grafana-values.yaml

@@ -26,6 +26,25 @@ grafana:
   service:
     port: 80
 
+  # p95 391M, observed max 1046M with no limit at all. Request is set at p95 so the
+  # scheduler sees reality; the limit is a manual exception above the 1.3x max
+  # formula, because a single query burst reached 1046M.
+  resources:
+    requests:
+      memory: "416Mi"
+      cpu: 100m
+    limits:
+      memory: "1536Mi"
+
+  # One block covers both the dashboards and datasources sidecars (p95 91M / 80M).
+  sidecar:
+    resources:
+      requests:
+        memory: "96Mi"
+        cpu: 10m
+      limits:
+        memory: "192Mi"
+
   additionalDataSources:
     - name: Loki
       type: loki
@@ -48,13 +67,21 @@ prometheus:
               storage: 40Gi
     resources:
       requests:
-        memory: "700Mi"
+        memory: "768Mi"
         cpu: 200m
       limits:
-        memory: "2Gi"
+        memory: "2560Mi"
 alertmanager:
   alertmanagerSpec:
     configSecret: alertmanager-config
+    # p95 66M, max 68M. Silences and notification state live here, so the request
+    # stays above p95 to keep the pod out of the eviction candidates.
+    resources:
+      requests:
+        memory: "96Mi"
+        cpu: 10m
+      limits:
+        memory: "192Mi"
     storage:
       volumeClaimTemplate:
         spec:
@@ -66,6 +93,54 @@ alertmanager:
             requests:
               storage: 20Gi
 
+# p95 103M, max 107M, and it grows with the number of cluster objects.
+kube-state-metrics:
+  # Values key is the dependency name from Chart.yaml, not the `kubeStateMetrics`
+  # condition key. Setting resources under `kubeStateMetrics:` is silently ignored.
+  resources:
+    requests:
+      memory: "128Mi"
+      cpu: 50m
+    limits:
+      memory: "256Mi"
+
+# p95 39M, max 40M. One per node, so it scales with node count.
+prometheus-node-exporter:
+  resources:
+    requests:
+      memory: "32Mi"
+      cpu: 20m
+    limits:
+      memory: "128Mi"
+
+# p95 75M, max 75M. Creates and reconciles every PrometheusRule in the cluster.
+prometheusOperator:
+  resources:
+    requests:
+      memory: "96Mi"
+      cpu: 50m
+    limits:
+      memory: "192Mi"
+
+# config-reloader sidecars (p95 33M, max 43M) are not covered: the chart does not
+# template `prometheusSpec.configReloader.resources`, so there is no values key
+# for them. They keep shipping with requests only.
+
 defaultRules:
   disabled:
     CPUThrottlingHigh: true
+    KubeControllerManagerDown: true
+    KubeSchedulerDown: true
+    KubeEtcdDown: true
+    KubeEtcdHighCommitDurations: true
+
+# k0s runs controller-manager/scheduler/etcd internally, not as pods with
+# component=kube-controller-manager/kube-scheduler/k8s-app=kube-etcd labels.
+# Their Services get no endpoints, so the targets are permanently down.
+# kube-proxy and kubelet have endpoints on k0s, keep them enabled.
+kubeControllerManager:
+  enabled: false
+kubeScheduler:
+  enabled: false
+kubeEtcd:
+  enabled: false

rackpeek/compose.yaml

@@ -0,0 +1,34 @@
+services:
+  rackpeek:
+    image: docker.io/aptacode/rackpeek:v2.1.0
+    container_name: rackpeek
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:8080:8080"
+    environment:
+      TZ: "Europe/Bratislava"
+    volumes:
+      - rackpeek-config:/app/config
+    networks:
+      - proxy
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.rackpeek.loadbalancer.server.port=8080"
+
+      # Local Router
+      - "traefik.http.routers.rackpeek-local.rule=Host(`rackpeek.workstation.internal`) || Host(`rack.workstation.internal`) || Host(`rackpeek.gigaforust.internal`) || Host(`rack.gigaforust.internal`)"
+      - "traefik.http.routers.rackpeek-local.entrypoints=websecure"
+      - "traefik.http.routers.rackpeek-local.tls=true"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"]
+      start_period: 15s
+      timeout: 5s
+      interval: 30s
+      retries: 3
+
+volumes:
+  rackpeek-config:
+
+networks:
+  proxy:
+    external: true

rackpeek/k8s/certificates.yaml

@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: rackpeek
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

rackpeek/k8s/ingress.yaml

@@ -0,0 +1,16 @@
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: rackpeek-local
+  namespace: rackpeek
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`rackpeek.workstation.internal`) || Host(`rack.workstation.internal`) || Host(`rackpeek.gigaforust.internal`) || Host(`rack.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: rackpeek-service
+          port: 8080
+  tls:
+    secretName: internal-wildcard-tls

rackpeek/k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rackpeek

rackpeek/k8s/rackpeek.yaml

@@ -0,0 +1,89 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: rackpeek-service
+  namespace: rackpeek
+spec:
+  selector:
+    app: rackpeek
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rackpeek-deployment
+  namespace: rackpeek
+  labels:
+    app: rackpeek
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rackpeek
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: rackpeek
+    spec:
+      securityContext:
+        # Image runs as uid/gid 1654
+        fsGroup: 1654
+      containers:
+        - name: rackpeek
+          image: docker.io/aptacode/rackpeek:v2.1.0
+          ports:
+            - name: http
+              containerPort: 8080
+          env:
+            - name: RPK_YAML_DIR
+              value: "/app/config"
+            - name: TZ
+              value: "Europe/Bratislava"
+          volumeMounts:
+            - name: rackpeek-config
+              mountPath: /app/config
+          startupProbe:
+            httpGet:
+              path: /health
+              port: http
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 20
+            periodSeconds: 30
+          resources:
+            requests:
+              memory: "160Mi"
+              cpu: "100m"
+            limits:
+              memory: "384Mi"
+              cpu: "500m"
+      volumes:
+        - name: rackpeek-config
+          persistentVolumeClaim:
+            claimName: rackpeek-pvc
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: rackpeek-pvc
+  namespace: rackpeek
+spec:
+  accessModes: ["ReadWriteOnce"]
+  storageClassName: local-path-retain
+  resources:
+    requests:
+      storage: 2Gi

reloader/k8s/reloader-values.yaml

@@ -0,0 +1,23 @@
+# Pinned chart: stakater/reloader 2.2.17 (app v1.4.22).
+# Deployed by the deploy workflow, namespace reloader.
+# Restarts pods when a ConfigMap or Secret they consume changes. Opt-in per workload
+# via the reloader.stakater.com/auto: "true" pod annotation; watchGlobally because
+# the workloads that need it are spread across a few dozen namespaces.
+
+reloader:
+  watchGlobally: true
+
+  deployment:
+    replicas: 1
+    # The chart defaults to no requests or limits, so the pod is evictable under node
+    # pressure and the restarts go with it.
+    # Memory was raised from 64Mi: measured p95 over 7 days is 73M, so the pod was
+    # running above its own request and sitting in the eviction candidates. This pod
+    # is the one that restarts every other pod, so it must not be evicted.
+    resources:
+      requests:
+        cpu: "10m"
+        memory: "96Mi"
+      limits:
+        cpu: "100m"
+        memory: "192Mi"

renovate.json

@@ -1,60 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": ["config:recommended"],
-  "enabledManagers": ["dockerfile", "docker-compose", "kubernetes", "helm-values", "custom.regex"],
-  "helm-values": {
-    "managerFilePatterns": ["/k8s/.+values\\.ya?ml$/"]
-  },
-  "kubernetes": {
-    "managerFilePatterns": ["/k8s/.+\\.ya?ml$/"]
-  },
-  "customManagers": [
-    {
-      "customType": "regex",
-      "description": "singlesource: playwright npm version pinned in npx command (k8s + compose)",
-      "fileMatch": ["^edu_master/k8s/playwright\\.yaml$", "^edu_master/compose\\.yaml$"],
-      "matchStrings": ["playwright@(?<currentValue>\\d+\\.\\d+\\.\\d+)"],
-      "datasourceTemplate": "npm",
-      "depNameTemplate": "playwright"
-    },
-    {
-      "customType": "regex",
-      "description": "singlesource: PLAYWRIGHT_VERSION file",
-      "fileMatch": ["^edu_master/PLAYWRIGHT_VERSION$"],
-      "matchStrings": ["^(?<currentValue>\\d+\\.\\d+\\.\\d+)$"],
-      "datasourceTemplate": "pypi",
-      "depNameTemplate": "playwright"
-    }
-  ],
-  "packageRules": [
-    {
-      "description": "singlesource playwright - use whichever version is found, keep docker+pypi+npm in sync",
-      "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
-      "groupName": "playwright singlesource",
-      "groupSlug": "playwright"
-    },
-    {
-      "description": "playwright must not automerge - version skew breaks WS handshake (checker.py:1523 vs playwright.yaml:20)",
-      "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
-      "automerge": false
-    },
-    {
-      "description": "Keep private homelab images unchanged",
-      "matchDatasources": ["docker"],
-      "matchPackageNames": ["/gcr\\.forust\\.xyz\\/forust\\/.+/"],
-      "enabled": false
-    },
-    {
-      "description": "Require approval for major upgrades",
-      "matchUpdateTypes": ["major"],
-      "dependencyDashboardApproval": true,
-      "automerge": false
-    },
-    {
-      "description": "Group container patch updates",
-      "matchDatasources": ["docker"],
-      "matchUpdateTypes": ["patch"],
-      "groupName": "container patch updates"
-    }
-  ]
-}

renovate/README.md

@@ -26,15 +26,17 @@ from Git and must be applied separately after every new cluster.
 
 Run it immediately instead of waiting for the six-hour schedule.
 
-Two options, both use the same `renovate/config.js`:
+Two options, both use the same `renovate/renovate.json`:
 
 ```sh
 kubectl create job --from=cronjob/renovate renovate-manual-$(date +%s) -n renovate
 ```
 
 or the `renovate-run` Actions workflow (Actions tab → `renovate-run` →
-Run workflow). It runs `renovate/renovate:44.103.0` on the self-hosted
-runner via Docker. Required Actions secrets (repo or org settings):
+Run workflow). It runs the same image as the CronJob on the self-hosted runner
+via Docker — the tag is read out of `renovate/k8s/cronjob.yaml` at run time
+rather than hardcoded, so the two cannot drift apart. Required Actions secrets
+(repo or org settings):
 
 - `RENOVATE_TOKEN` — renovate-bot PAT (repository + issue read/write).
 - `RENOVATE_GITHUB_COM_TOKEN` — optional, for changelogs and GitHub rate limits.
@@ -64,6 +66,33 @@ docker compose -f renovate-compose.yaml run --rm renovate
 The Compose file is intentionally named `renovate-compose.yaml`, so the
 repository's automatic deployment discovery does not start it accidentally.
 
+## Configuration
+
+`renovate/renovate.json` is the single source of truth. The Compose file and the
+`renovate-run` workflow mount that file directly.
+
+A ConfigMap cannot read from the repository, so the CronJob needs the config
+inlined. `renovate/k8s/configmap.yaml` is therefore a **generated** copy:
+
+```sh
+.gitea/workflows/sync-renovate-configmap.sh          # regenerate after editing
+.gitea/workflows/sync-renovate-configmap.sh --check  # fail if out of date
+```
+
+The `renovate-ci` workflow runs the `--check` form on every PR and push, so a
+config edit that forgets to regenerate the ConfigMap cannot be merged.
+
+Beyond images, `customManagers` in the config track:
+
+- Helm chart versions pinned in `.gitea/workflows/deploy-lib.sh`. The built-in
+  `helmv3` manager only reads `Chart.yaml` and `helm-values` only reads values
+  files, so neither sees a version written into a `helm upgrade` command —
+  these are declared as `custom.regex` managers against the `helm` datasource.
+- CI linter versions in `.gitea/workflows/tool-versions.env`.
+
+The Renovate image tag is deliberately _not_ in `tool-versions.env`:
+`renovate/k8s/cronjob.yaml` owns it, and the workflows read it from there.
+
 ## How updates flow
 
 Renovate scans both `compose.yaml` files and Kubernetes manifests, opens a

renovate/config.js

@@ -1,44 +0,0 @@
-module.exports = {
-  platform: 'gitea',
-  endpoint: process.env.RENOVATE_ENDPOINT || 'https://gitea.forust.xyz/api/v1',
-  enabledManagers: ['docker-compose', 'kubernetes', 'helm-values'],
-  'helm-values': {
-    managerFilePatterns: ['/k8s/.+values\\.ya?ml$/'],
-  },
-  kubernetes: {
-    managerFilePatterns: ['/k8s/.+\\.ya?ml$/'],
-  },
-  repositories: (process.env.RENOVATE_REPOSITORIES || '')
-    .split(',')
-    .map((repository) => repository.trim())
-    .filter(Boolean),
-  onboarding: false,
-  requireConfig: 'optional',
-  autodiscover: false,
-  dependencyDashboard: true,
-  prCreation: 'immediate',
-  labels: ['dependencies', 'automated'],
-  extends: [
-    'config:recommended',
-    ':dependencyDashboard',
-  ],
-  packageRules: [
-    {
-      description: 'Do not update private homelab images',
-      matchDatasources: ['docker'],
-      matchPackageNames: ['/gcr\\.forust\\.xyz\\/forust\\/.+/'],
-      enabled: false,
-    },
-    {
-      description: 'Keep major upgrades manual',
-      matchUpdateTypes: ['major'],
-      dependencyDashboardApproval: true,
-      automerge: false,
-    },
-    {
-      description: 'Group patch updates',
-      matchUpdateTypes: ['patch'],
-      groupName: 'container patch updates',
-    },
-  ],
-};

renovate/k8s/configmap.yaml

@@ -1,51 +1,211 @@
+# GENERATED FILE - do not edit by hand.
+# Source: renovate/renovate.json
+# Regenerate: .gitea/workflows/sync-renovate-configmap.sh
+# Verify:     .gitea/workflows/sync-renovate-configmap.sh --check
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: renovate-config
   namespace: renovate
 data:
-  config.js: |
-    module.exports = {
-      platform: 'gitea',
-      endpoint: process.env.RENOVATE_ENDPOINT || 'https://gitea.forust.xyz/api/v1',
-      enabledManagers: ['docker-compose', 'kubernetes', 'helm-values'],
-      'helm-values': {
-        managerFilePatterns: ['/k8s/.+values\\.ya?ml$/'],
+  renovate.json: |
+    {
+      "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+      "extends": ["config:recommended", ":dependencyDashboard"],
+      "enabledManagers": ["dockerfile", "docker-compose", "kubernetes", "helm-values", "custom.regex"],
+      "onboarding": false,
+      "requireConfig": "optional",
+      "autodiscover": false,
+      "dependencyDashboard": true,
+      "prCreation": "immediate",
+      "labels": ["dependencies", "automated"],
+      "helm-values": {
+        "managerFilePatterns": ["/k8s/.+values\\.ya?ml$/"]
       },
-      kubernetes: {
-        managerFilePatterns: ['/k8s/.+\\.ya?ml$/'],
+      "kubernetes": {
+        "managerFilePatterns": ["/k8s/.+\\.ya?ml$/"]
       },
-      repositories: (process.env.RENOVATE_REPOSITORIES || '')
-        .split(',')
-        .map((repository) => repository.trim())
-        .filter(Boolean),
-      onboarding: false,
-      requireConfig: 'optional',
-      autodiscover: false,
-      dependencyDashboard: true,
-      prCreation: 'immediate',
-      labels: ['dependencies', 'automated'],
-      extends: [
-        'config:recommended',
-        ':dependencyDashboard',
+      "customManagers": [
+        {
+          "customType": "regex",
+          "description": "singlesource: playwright npm version pinned in npx command (k8s + compose)",
+          "managerFilePatterns": ["^edu_master/k8s/playwright\\.yaml$", "^edu_master/compose\\.yaml$"],
+          "matchStrings": ["playwright@(?<currentValue>\\d+\\.\\d+\\.\\d+)"],
+          "datasourceTemplate": "npm",
+          "depNameTemplate": "playwright"
+        },
+        {
+          "customType": "regex",
+          "description": "singlesource: PLAYWRIGHT_VERSION file",
+          "managerFilePatterns": ["^edu_master/PLAYWRIGHT_VERSION$"],
+          "matchStrings": ["^(?<currentValue>\\d+\\.\\d+\\.\\d+)$"],
+          "datasourceTemplate": "pypi",
+          "depNameTemplate": "playwright"
+        },
+        {
+          "customType": "regex",
+          "description": "kube-prometheus-stack chart version pinned in the deploy workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+          "matchStrings": ["\\|prometheus-community/kube-prometheus-stack\\|prometheus\\|(?<currentValue>[0-9.]+)\\|"],
+          "datasourceTemplate": "helm",
+          "depNameTemplate": "kube-prometheus-stack",
+          "registryUrlTemplate": "https://prometheus-community.github.io/helm-charts"
+        },
+        {
+          "customType": "regex",
+          "description": "grafana/loki chart version pinned in the deploy workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+          "matchStrings": ["\\|grafana/loki\\|prometheus\\|(?<currentValue>[0-9.]+)\\|"],
+          "datasourceTemplate": "helm",
+          "depNameTemplate": "loki",
+          "registryUrlTemplate": "https://grafana.github.io/helm-charts"
+        },
+        {
+          "customType": "regex",
+          "description": "grafana/alloy chart version pinned in the deploy workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+          "matchStrings": ["\\|grafana/alloy\\|prometheus\\|(?<currentValue>[0-9.]+)\\|"],
+          "datasourceTemplate": "helm",
+          "depNameTemplate": "alloy",
+          "registryUrlTemplate": "https://grafana.github.io/helm-charts"
+        },
+        {
+          "customType": "regex",
+          "description": "actionlint version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)ACTIONLINT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "github-tags",
+          "depNameTemplate": "rhysd/actionlint"
+        },
+        {
+          "customType": "regex",
+          "description": "shellcheck version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)SHELLCHECK_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "github-tags",
+          "depNameTemplate": "koalaman/shellcheck"
+        },
+        {
+          "customType": "regex",
+          "description": "kubeconform version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)KUBECONFORM_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "github-tags",
+          "depNameTemplate": "yannh/kubeconform"
+        },
+        {
+          "customType": "regex",
+          "description": "uv version used to build the pytest venv",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)UV_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "github-tags",
+          "depNameTemplate": "astral-sh/uv"
+        },
+        {
+          "customType": "regex",
+          "description": "prettier version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)PRETTIER_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "npm",
+          "depNameTemplate": "prettier"
+        },
+        {
+          "customType": "regex",
+          "description": "ruff version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)RUFF_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "pypi",
+          "depNameTemplate": "ruff"
+        },
+        {
+          "customType": "regex",
+          "description": "pip-audit version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)PIP_AUDIT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "pypi",
+          "depNameTemplate": "pip-audit"
+        },
+        {
+          "customType": "regex",
+          "description": "yamllint version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)YAMLLINT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "pypi",
+          "depNameTemplate": "yamllint"
+        },
+        {
+          "customType": "regex",
+          "description": "hadolint version used by the ci workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)HADOLINT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "github-tags",
+          "depNameTemplate": "hadolint/hadolint"
+        },
+        {
+          "customType": "regex",
+          "description": "node version the ci workflow runs npm with",
+          "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+          "matchStrings": ["(?:^|\\n)NODE_VERSION=\"(?<currentValue>[0-9.]+)\""],
+          "datasourceTemplate": "node",
+          "depNameTemplate": "node"
+        },
+        {
+          "customType": "regex",
+          "description": "stakater/reloader chart version pinned in the deploy workflow",
+          "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+          "matchStrings": ["\\|stakater/reloader\\|reloader\\|(?<currentValue>[0-9.]+)\\|"],
+          "datasourceTemplate": "helm",
+          "depNameTemplate": "reloader",
+          "registryUrlTemplate": "https://stakater.github.io/stakater-charts"
+        }
       ],
-      packageRules: [
+      "packageRules": [
         {
-          description: 'Do not update private homelab images',
-          matchDatasources: ['docker'],
-          matchPackageNames: ['/gcr\\.forust\\.xyz\\/forust\\/.+/'],
-          enabled: false,
+          "description": "Keep private homelab images unchanged",
+          "matchDatasources": ["docker"],
+          "matchPackageNames": ["/gcr\\.forust\\.xyz\\/forust\\/.+/"],
+          "enabled": false
         },
         {
-          description: 'Keep major upgrades manual',
-          matchUpdateTypes: ['major'],
-          dependencyDashboardApproval: true,
-          automerge: false,
+          "description": "singlesource playwright - use whichever version is found, keep docker+pypi+npm in sync",
+          "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
+          "groupName": "playwright singlesource",
+          "groupSlug": "playwright"
         },
         {
-          description: 'Group patch updates',
-          matchUpdateTypes: ['patch'],
-          groupName: 'container patch updates',
+          "description": "playwright must not automerge - version skew breaks the WS handshake (checker.py:1523 vs playwright.yaml:20)",
+          "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
+          "automerge": false
         },
-      ],
-    };
+        {
+          "description": "Renovate updates itself in lockstep across the CronJob and the Compose file",
+          "matchPackageNames": ["renovate/renovate"],
+          "groupName": "renovate self-update",
+          "automerge": false
+        },
+        {
+          "description": "CI runs npm on the node the panel image is built from - the NODE_VERSION pin in tool-versions.env and node:22-alpine in the Dockerfile are the same dependency and move as one",
+          "matchPackageNames": ["node"],
+          "groupName": "node runtime",
+          "groupSlug": "node",
+          "automerge": false
+        },
+        {
+          "description": "Helm chart bumps change PVC fields and admission behaviour, keep them reviewable",
+          "matchDatasources": ["helm"],
+          "automerge": false
+        },
+        {
+          "description": "Require approval for major upgrades",
+          "matchUpdateTypes": ["major"],
+          "dependencyDashboardApproval": true,
+          "automerge": false
+        },
+        {
+          "description": "Group container patch updates",
+          "matchDatasources": ["docker"],
+          "matchUpdateTypes": ["patch"],
+          "groupName": "container patch updates"
+        }
+      ]
+    }

renovate/k8s/cronjob.yaml

@@ -10,13 +10,16 @@ spec:
   failedJobsHistoryLimit: 3
   jobTemplate:
     spec:
+      # Reap finished job pods (including manual `create job --from` runs,
+      # which history limits never delete). Keeps a day for debugging.
+      ttlSecondsAfterFinished: 86400
       backoffLimit: 1
       template:
         spec:
           restartPolicy: Never
           containers:
             - name: renovate
-              image: renovate/renovate:44.106.0
+              image: renovate/renovate:44.115.13
               env:
                 - name: RENOVATE_PLATFORM
                   value: gitea
@@ -36,7 +39,7 @@ spec:
                       name: renovate-secrets
                       key: RENOVATE_REPOSITORIES
                 - name: RENOVATE_CONFIG_FILE
-                  value: /opt/renovate/config.js
+                  value: /opt/renovate/renovate.json
                 - name: RENOVATE_BASE_DIR
                   value: /tmp/renovate
                 - name: RENOVATE_GITHUB_COM_TOKEN
@@ -49,8 +52,8 @@ spec:
                   value: info
               volumeMounts:
                 - name: config
-                  mountPath: /opt/renovate/config.js
-                  subPath: config.js
+                  mountPath: /opt/renovate/renovate.json
+                  subPath: renovate.json
                   readOnly: true
           volumes:
             - name: config

renovate/renovate-compose.yaml

@@ -1,6 +1,8 @@
 services:
   renovate:
-    image: renovate/renovate:44.103.0
+    # Kept in step with renovate/k8s/cronjob.yaml by the "renovate self-update"
+    # package rule in renovate/renovate.json.
+    image: renovate/renovate:44.115.9
     container_name: renovate
     restart: "no"
     env_file:
@@ -10,8 +12,8 @@ services:
       RENOVATE_ENDPOINT: ${RENOVATE_ENDPOINT:?set RENOVATE_ENDPOINT}
       RENOVATE_TOKEN: ${RENOVATE_TOKEN:?set RENOVATE_TOKEN}
       RENOVATE_REPOSITORIES: ${RENOVATE_REPOSITORIES:?set RENOVATE_REPOSITORIES}
-      RENOVATE_CONFIG_FILE: /opt/renovate/config.js
+      RENOVATE_CONFIG_FILE: /opt/renovate/renovate.json
       RENOVATE_BASE_DIR: /tmp/renovate
       LOG_LEVEL: ${LOG_LEVEL:-info}
     volumes:
-      - ./config.js:/opt/renovate/config.js:ro
+      - ./renovate.json:/opt/renovate/renovate.json:ro

renovate/renovate.json

@@ -0,0 +1,200 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["config:recommended", ":dependencyDashboard"],
+  "enabledManagers": ["dockerfile", "docker-compose", "kubernetes", "helm-values", "custom.regex"],
+  "onboarding": false,
+  "requireConfig": "optional",
+  "autodiscover": false,
+  "dependencyDashboard": true,
+  "prCreation": "immediate",
+  "labels": ["dependencies", "automated"],
+  "helm-values": {
+    "managerFilePatterns": ["/k8s/.+values\\.ya?ml$/"]
+  },
+  "kubernetes": {
+    "managerFilePatterns": ["/k8s/.+\\.ya?ml$/"]
+  },
+  "customManagers": [
+    {
+      "customType": "regex",
+      "description": "singlesource: playwright npm version pinned in npx command (k8s + compose)",
+      "managerFilePatterns": ["^edu_master/k8s/playwright\\.yaml$", "^edu_master/compose\\.yaml$"],
+      "matchStrings": ["playwright@(?<currentValue>\\d+\\.\\d+\\.\\d+)"],
+      "datasourceTemplate": "npm",
+      "depNameTemplate": "playwright"
+    },
+    {
+      "customType": "regex",
+      "description": "singlesource: PLAYWRIGHT_VERSION file",
+      "managerFilePatterns": ["^edu_master/PLAYWRIGHT_VERSION$"],
+      "matchStrings": ["^(?<currentValue>\\d+\\.\\d+\\.\\d+)$"],
+      "datasourceTemplate": "pypi",
+      "depNameTemplate": "playwright"
+    },
+    {
+      "customType": "regex",
+      "description": "kube-prometheus-stack chart version pinned in the deploy workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+      "matchStrings": ["\\|prometheus-community/kube-prometheus-stack\\|prometheus\\|(?<currentValue>[0-9.]+)\\|"],
+      "datasourceTemplate": "helm",
+      "depNameTemplate": "kube-prometheus-stack",
+      "registryUrlTemplate": "https://prometheus-community.github.io/helm-charts"
+    },
+    {
+      "customType": "regex",
+      "description": "grafana/loki chart version pinned in the deploy workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+      "matchStrings": ["\\|grafana/loki\\|prometheus\\|(?<currentValue>[0-9.]+)\\|"],
+      "datasourceTemplate": "helm",
+      "depNameTemplate": "loki",
+      "registryUrlTemplate": "https://grafana.github.io/helm-charts"
+    },
+    {
+      "customType": "regex",
+      "description": "grafana/alloy chart version pinned in the deploy workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+      "matchStrings": ["\\|grafana/alloy\\|prometheus\\|(?<currentValue>[0-9.]+)\\|"],
+      "datasourceTemplate": "helm",
+      "depNameTemplate": "alloy",
+      "registryUrlTemplate": "https://grafana.github.io/helm-charts"
+    },
+    {
+      "customType": "regex",
+      "description": "actionlint version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)ACTIONLINT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "github-tags",
+      "depNameTemplate": "rhysd/actionlint"
+    },
+    {
+      "customType": "regex",
+      "description": "shellcheck version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)SHELLCHECK_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "github-tags",
+      "depNameTemplate": "koalaman/shellcheck"
+    },
+    {
+      "customType": "regex",
+      "description": "kubeconform version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)KUBECONFORM_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "github-tags",
+      "depNameTemplate": "yannh/kubeconform"
+    },
+    {
+      "customType": "regex",
+      "description": "uv version used to build the pytest venv",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)UV_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "github-tags",
+      "depNameTemplate": "astral-sh/uv"
+    },
+    {
+      "customType": "regex",
+      "description": "prettier version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)PRETTIER_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "npm",
+      "depNameTemplate": "prettier"
+    },
+    {
+      "customType": "regex",
+      "description": "ruff version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)RUFF_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "pypi",
+      "depNameTemplate": "ruff"
+    },
+    {
+      "customType": "regex",
+      "description": "pip-audit version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)PIP_AUDIT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "pypi",
+      "depNameTemplate": "pip-audit"
+    },
+    {
+      "customType": "regex",
+      "description": "yamllint version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)YAMLLINT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "pypi",
+      "depNameTemplate": "yamllint"
+    },
+    {
+      "customType": "regex",
+      "description": "hadolint version used by the ci workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)HADOLINT_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "github-tags",
+      "depNameTemplate": "hadolint/hadolint"
+    },
+    {
+      "customType": "regex",
+      "description": "node version the ci workflow runs npm with",
+      "managerFilePatterns": ["^\\.gitea/workflows/tool-versions\\.env$"],
+      "matchStrings": ["(?:^|\\n)NODE_VERSION=\"(?<currentValue>[0-9.]+)\""],
+      "datasourceTemplate": "node",
+      "depNameTemplate": "node"
+    },
+    {
+      "customType": "regex",
+      "description": "stakater/reloader chart version pinned in the deploy workflow",
+      "managerFilePatterns": ["^\\.gitea/workflows/deploy-lib\\.sh$"],
+      "matchStrings": ["\\|stakater/reloader\\|reloader\\|(?<currentValue>[0-9.]+)\\|"],
+      "datasourceTemplate": "helm",
+      "depNameTemplate": "reloader",
+      "registryUrlTemplate": "https://stakater.github.io/stakater-charts"
+    }
+  ],
+  "packageRules": [
+    {
+      "description": "Keep private homelab images unchanged",
+      "matchDatasources": ["docker"],
+      "matchPackageNames": ["/gcr\\.forust\\.xyz\\/forust\\/.+/"],
+      "enabled": false
+    },
+    {
+      "description": "singlesource playwright - use whichever version is found, keep docker+pypi+npm in sync",
+      "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
+      "groupName": "playwright singlesource",
+      "groupSlug": "playwright"
+    },
+    {
+      "description": "playwright must not automerge - version skew breaks the WS handshake (checker.py:1523 vs playwright.yaml:20)",
+      "matchPackageNames": ["playwright", "mcr.microsoft.com/playwright"],
+      "automerge": false
+    },
+    {
+      "description": "Renovate updates itself in lockstep across the CronJob and the Compose file",
+      "matchPackageNames": ["renovate/renovate"],
+      "groupName": "renovate self-update",
+      "automerge": false
+    },
+    {
+      "description": "CI runs npm on the node the panel image is built from - the NODE_VERSION pin in tool-versions.env and node:22-alpine in the Dockerfile are the same dependency and move as one",
+      "matchPackageNames": ["node"],
+      "groupName": "node runtime",
+      "groupSlug": "node",
+      "automerge": false
+    },
+    {
+      "description": "Helm chart bumps change PVC fields and admission behaviour, keep them reviewable",
+      "matchDatasources": ["helm"],
+      "automerge": false
+    },
+    {
+      "description": "Require approval for major upgrades",
+      "matchUpdateTypes": ["major"],
+      "dependencyDashboardApproval": true,
+      "automerge": false
+    },
+    {
+      "description": "Group container patch updates",
+      "matchDatasources": ["docker"],
+      "matchUpdateTypes": ["patch"],
+      "groupName": "container patch updates"
+    }
+  ]
+}

searxng/compose.yaml

@@ -3,7 +3,7 @@
 services:
   core:
     container_name: searxng-core
-    image: docker.io/searxng/searxng:${SEARXNG_VERSION:-2026.09.13-d4ce87c23}
+    image: docker.io/searxng/searxng:${SEARXNG_VERSION:-2026.9.25-12f8b6515}
     restart: unless-stopped
     # ports:
     # - ${SEARXNG_PORT:-8080}

searxng/k8s/searxng.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: searxng
-          image: docker.io/searxng/searxng:2026.09.13-d4ce87c23
+          image: docker.io/searxng/searxng:2026.9.25-12f8b6515
           envFrom:
             - configMapRef:
                 name: searxng-config
@@ -38,12 +38,13 @@ spec:
           volumeMounts:
             - name: cache
               mountPath: /var/cache/searxng
+          # p95 134M, max 145M over 7 days. Was 300Mi/700Mi.
           resources:
             requests:
-              memory: "300Mi"
+              memory: "160Mi"
               cpu: "30m"
             limits:
-              memory: "700Mi"
+              memory: "512Mi"
               cpu: "500m"
       volumes:
         - name: cache