fix(deploy): resolve Compose config and check namespaced pod Secrets

.gitea/tests/deploy-validation.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Local regressions only: kubectl is mocked and Docker is used for config parsing.
+set -euo pipefail
+repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+scratch="$(mktemp -d)"
+trap 'rm -rf "$scratch"' EXIT
+
+mkdir -p "$scratch/repo/app" "$scratch/repo/postgres" "$scratch/repo/netbird" "$scratch/repo/renovate"
+git -C "$scratch/repo" init -q
+for file in app/compose.yaml postgres/shared-compose.yaml netbird/client.compose.yaml renovate/renovate-compose.yaml; do
+  touch "$scratch/repo/$file"
+done
+git -C "$scratch/repo" add .
+# shellcheck source=../workflows/compose-lint.sh
+source "$repo/.gitea/workflows/compose-lint.sh"
+actual="$(cd "$scratch/repo" && compose_files)"
+expected=$'app/compose.yaml\nnetbird/client.compose.yaml\npostgres/shared-compose.yaml\nrenovate/renovate-compose.yaml'
+[ "$actual" = "$expected" ] || { echo 'Compose discovery missed a file' >&2; exit 1; }
+
+cat >"$scratch/compose.yaml" <<'YAML'
+services:
+  example:
+    image: busybox:1.37.0
+    environment:
+      REQUIRED: ${HOMELAB_TEST_REQUIRED:?required for this regression}
+YAML
+unset HOMELAB_TEST_REQUIRED
+if validate_compose_file "$scratch/compose.yaml" >"$scratch/config.log" 2>&1; then
+  echo 'Full Compose validation accepted a missing variable' >&2
+  exit 1
+fi
+rg -q 'required for this regression' "$scratch/config.log"
+HOMELAB_TEST_REQUIRED=present validate_compose_file "$scratch/compose.yaml"
+
+cat >"$scratch/resources.json" <<'JSON'
+{"kind":"List","items":[
+  {"kind":"Deployment","metadata":{"namespace":"app"},"spec":{"template":{"spec":{
+    "containers":[{"envFrom":[{"secretRef":{"name":"credentials"}},{"secretRef":{"name":"optional","optional":true}}],"env":[{"valueFrom":{"secretKeyRef":{"name":"credentials","key":"password"}}}]}],
+    "initContainers":[{"envFrom":[{"secretRef":{"name":"init"}}]}],
+    "imagePullSecrets":[{"name":"registry"}],
+    "volumes":[{"secret":{"secretName":"mounted"}},{"projected":{"sources":[{"secret":{"name":"projected"}},{"secret":{"name":"optional-projected","optional":true}}]}}]
+  }}}},
+  {"kind":"CronJob","metadata":{},"spec":{"jobTemplate":{"spec":{"template":{"spec":{"containers":[{"envFrom":[{"secretRef":{"name":"cron"}}]}]}}}}}},
+  {"kind":"IngressRoute","metadata":{"namespace":"app"},"spec":{"tls":{"secretName":"controller-issued-tls"}}}
+]}
+JSON
+actual="$(jq -r -f "$repo/.gitea/workflows/secret-references.jq" "$scratch/resources.json" | sort)"
+expected=$'app credentials\napp init\napp mounted\napp projected\napp registry\ndefault cron'
+[ "$actual" = "$expected" ] || { echo "Unexpected Secret references: $actual" >&2; exit 1; }
+
+REPO="$repo"
+# shellcheck source=../workflows/deploy-lib.sh
+source "$repo/.gitea/workflows/deploy-lib.sh"
+K8S_MANIFESTS=("$scratch/resources.json")
+KUSTOMIZE_APPS=()
+# No live cluster access. Reject credentials in app even if they exist elsewhere.
+kubectl() {
+  case "$1" in
+    create) cat "$scratch/resources.json" ;;
+    get)
+      if [ "$3" = credentials ] && [ "$5" = app ]; then
+        return 1
+      fi
+      return 0
+      ;;
+    *) echo "Unexpected kubectl invocation: $*" >&2; return 1 ;;
+  esac
+}
+if check_referenced_secrets >"$scratch/secrets.log"; then
+  echo 'Namespace-scoped Secret check accepted a missing Secret' >&2
+  exit 1
+fi
+rg -q 'MISSING OR UNREADABLE: app/credentials' "$scratch/secrets.log"
+# API/rendering errors must not produce an empty reference list and pass.
+kubectl() { return 1; }
+if check_referenced_secrets >"$scratch/secrets.log"; then
+  echo 'Secret check accepted a failed manifest render' >&2
+  exit 1
+fi
+printf '%s\n' 'Deploy validation regressions passed.'

.gitea/workflows/ci.yaml

@@ -98,6 +98,7 @@ jobs:
             exit 0
           fi
           shellcheck --external-sources --source-path=SCRIPTDIR --severity=style "${scripts[@]}"
+          bash .gitea/tests/deploy-validation.sh
 
   lint-prettier:
     runs-on: [self-hosted, linux, arch, homelab]

.gitea/workflows/compose-lint.sh

@@ -21,8 +21,7 @@
 # All committed Compose files, including the ones deploy never starts.
 compose_files() {
   git ls-files \
-    '*/compose.yaml' '*/compose.yml' 'compose.yaml' 'compose.yml' \
-    '*/docker-compose.yaml' '*/docker-compose.yml'
+    '*compose.yaml' '*compose.yml'
 }
 
 # Prints the flags that turn `docker compose config` into the general check.

.gitea/workflows/deploy-lib.sh

@@ -686,27 +686,52 @@ stage_preflight() {
   git -C "$REPO" reset --hard "$target"
 }
 
+# Required pod Secrets, scoped to the resource namespace. TLS route Secrets are
+# created by cert-manager and are not prerequisites for applying a Certificate.
+check_referenced_secrets() {
+  local m k objects refs extracted ns name
+  local missing=()
+  refs=""
+  for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do
+    objects="$(kubectl create --dry-run=client --validate=false -f "$m" -o json)" || return 1
+    extracted="$(printf '%s' "$objects" | jq -r -f "$REPO/.gitea/workflows/secret-references.jq")" || return 1
+    refs+="$extracted"$'\n'
+  done
+  for k in ${KUSTOMIZE_APPS[@]+"${KUSTOMIZE_APPS[@]}"}; do
+    objects="$(kubectl kustomize "$k" | kubectl create --dry-run=client --validate=false -f - -o json)" || return 1
+    extracted="$(printf '%s' "$objects" | jq -r -f "$REPO/.gitea/workflows/secret-references.jq")" || return 1
+    refs+="$extracted"$'\n'
+  done
+  while read -r ns name; do
+    [ -n "${name:-}" ] || continue
+    if kubectl get secret "$name" -n "$ns" -o name >/dev/null 2>&1; then
+      echo "  ok: $ns/$name"
+    else
+      echo "  MISSING OR UNREADABLE: $ns/$name"
+      missing+=("$ns/$name")
+    fi
+  done < <(printf '%s' "$refs" | sort -u)
+  if [ "${#missing[@]}" -gt 0 ]; then
+    echo "ERROR: required pod Secrets are missing or unreadable:"
+    printf '  - %s\n' "${missing[@]}"
+    echo "Create them in the listed namespaces from the service's secret example."
+    return 1
+  fi
+}
+
 stage_validate() {
   cd "$REPO"
   select_manifests
   local m k cf
-  # Compose .env files and secret files are gitignored by design, so the
-  # workstation never has real values for the inactive stacks. This stage only
-  # runs the full check on active stacks; the general structure check for every
-  # committed Compose file (active or not) lives in the ci workflow, which has no
-  # .env at all.
-  #
-  # Active stacks are still validated with interpolation and env-file resolution
-  # off, so required-variable guards (:?) and missing local files do not fail the
-  # deploy. Normalization and consistency checks stay enabled.
+  # The deploy host has the local .env and secret files. Resolve them here so
+  # missing configuration fails before either apply job changes workloads.
+  # CI keeps the structure-only check for inactive stacks.
   # shellcheck source=compose-lint.sh
   source "$REPO/.gitea/workflows/compose-lint.sh"
-  local compose_validate_flags=()
-  mapfile -t compose_validate_flags < <(compose_safe_flags)
   log "Validate compose stacks"
   for cf in ${COMPOSE_STACKS[@]+"${COMPOSE_STACKS[@]}"}; do
     echo "  config: $cf"
-    validate_compose_file "$cf" ${compose_validate_flags[@]+"${compose_validate_flags[@]}"}
+    validate_compose_file "$cf"
   done
   log "Validate k8s manifests (kubectl dry-run=client)"
   for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do
@@ -724,33 +749,7 @@ stage_validate() {
   done
   log "Checking referenced Secrets exist"
   echo "  (deploy never applies *secret*.yaml; create missing ones manually)"
-  local ref_secrets=() missing_secrets=() all_secrets s
-  if [ "${#K8S_MANIFESTS[@]}" -gt 0 ]; then
-    while IFS= read -r s; do
-      [ -n "$s" ] && ref_secrets+=("$s")
-    done < <(
-      {
-        grep -h -A1 -E 'secretRef:|secretKeyRef:' "${K8S_MANIFESTS[@]}" 2>/dev/null || true
-        grep -h -E 'secretName:' "${K8S_MANIFESTS[@]}" 2>/dev/null || true
-      } | grep -E 'name:' | sed -E 's/.*name:[[:space:]]*//' | tr -d '"'"'"' "'"'" | sed -E 's/[[:space:]]*#.*//' | awk 'NF' | sort -u || true
-    )
-  fi
-  all_secrets="$(kubectl get secrets -A --no-headers -o custom-columns=:metadata.name 2>/dev/null || true)"
-  for s in ${ref_secrets[@]+"${ref_secrets[@]}"}; do
-    if printf '%s\n' "$all_secrets" | grep -qx "$s"; then
-      echo "  ok: $s"
-    else
-      echo "  MISSING: $s"
-      missing_secrets+=("$s")
-    fi
-  done
-  if [ "${#missing_secrets[@]}" -gt 0 ]; then
-    echo "ERROR: ${#missing_secrets[@]} referenced Secret(s) not found in the cluster:"
-    printf '  - %s\n' "${missing_secrets[@]}"
-    echo "Create them manually from the laptop, e.g.:"
-    echo "  kubectl apply -f SERVICE/k8s/secrets.yaml  # see SERVICE/k8s/secrets.yaml.example"
-    exit 1
-  fi
+  check_referenced_secrets
 }
 
 stage_apply_k8s() {

.gitea/workflows/secret-references.jq

@@ -0,0 +1,13 @@
+# kubectl emits a List for files containing multiple resources.
+(if .kind == "List" then .items[] else . end)
+| (.metadata.namespace // "default") as $ns
+| [
+    (.. | objects
+      | (.secretRef? // empty), (.secretKeyRef? // empty), (.secret? // empty)
+      | select(.optional != true)
+      | .name // .secretName // empty),
+    (.. | objects | .imagePullSecrets[]?.name)
+  ]
+| unique[]
+| select(. != null and . != "")
+| "\($ns) \(.)"