diff --git a/.gitea/tests/deploy-validation.sh b/.gitea/tests/deploy-validation.sh new file mode 100755 index 0000000..94e6480 --- /dev/null +++ b/.gitea/tests/deploy-validation.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Local regressions only: kubectl is mocked and Docker is used for config parsing. +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT + +mkdir -p "$scratch/repo/app" "$scratch/repo/postgres" "$scratch/repo/netbird" "$scratch/repo/renovate" +git -C "$scratch/repo" init -q +for file in app/compose.yaml postgres/shared-compose.yaml netbird/client.compose.yaml renovate/renovate-compose.yaml; do + touch "$scratch/repo/$file" +done +git -C "$scratch/repo" add . +# shellcheck source=../workflows/compose-lint.sh +source "$repo/.gitea/workflows/compose-lint.sh" +actual="$(cd "$scratch/repo" && compose_files)" +expected=$'app/compose.yaml\nnetbird/client.compose.yaml\npostgres/shared-compose.yaml\nrenovate/renovate-compose.yaml' +[ "$actual" = "$expected" ] || { echo 'Compose discovery missed a file' >&2; exit 1; } + +cat >"$scratch/compose.yaml" <<'YAML' +services: + example: + image: busybox:1.37.0 + environment: + REQUIRED: ${HOMELAB_TEST_REQUIRED:?required for this regression} +YAML +unset HOMELAB_TEST_REQUIRED +if validate_compose_file "$scratch/compose.yaml" >"$scratch/config.log" 2>&1; then + echo 'Full Compose validation accepted a missing variable' >&2 + exit 1 +fi +grep -q 'required for this regression' "$scratch/config.log" +HOMELAB_TEST_REQUIRED=present validate_compose_file "$scratch/compose.yaml" + +cat >"$scratch/resources.json" <<'JSON' +{"kind":"List","items":[ + {"kind":"Deployment","metadata":{"namespace":"app"},"spec":{"template":{"spec":{ + "containers":[{"envFrom":[{"secretRef":{"name":"credentials"}},{"secretRef":{"name":"optional","optional":true}}],"env":[{"valueFrom":{"secretKeyRef":{"name":"credentials","key":"password"}}}]}], + "initContainers":[{"envFrom":[{"secretRef":{"name":"init"}}]}], + "imagePullSecrets":[{"name":"registry"}], + "volumes":[{"secret":{"secretName":"mounted"}},{"projected":{"sources":[{"secret":{"name":"projected"}},{"secret":{"name":"optional-projected","optional":true}}]}}] + }}}}, + {"kind":"CronJob","metadata":{},"spec":{"jobTemplate":{"spec":{"template":{"spec":{"containers":[{"envFrom":[{"secretRef":{"name":"cron"}}]}]}}}}}}, + {"kind":"IngressRoute","metadata":{"namespace":"app"},"spec":{"tls":{"secretName":"controller-issued-tls"}}} +]} +JSON +actual="$(jq -r -f "$repo/.gitea/workflows/secret-references.jq" "$scratch/resources.json" | sort)" +expected=$'app credentials\napp init\napp mounted\napp projected\napp registry\ndefault cron' +[ "$actual" = "$expected" ] || { echo "Unexpected Secret references: $actual" >&2; exit 1; } + +REPO="$repo" +# shellcheck source=../workflows/deploy-lib.sh +source "$repo/.gitea/workflows/deploy-lib.sh" +K8S_MANIFESTS=("$scratch/resources.json") +KUSTOMIZE_APPS=() +# No live cluster access. Reject credentials in app even if they exist elsewhere. +kubectl() { + case "$1" in + create) cat "$scratch/resources.json" ;; + get) + if [ "$3" = credentials ] && [ "$5" = app ]; then + return 1 + fi + return 0 + ;; + *) echo "Unexpected kubectl invocation: $*" >&2; return 1 ;; + esac +} +if check_referenced_secrets >"$scratch/secrets.log"; then + echo 'Namespace-scoped Secret check accepted a missing Secret' >&2 + exit 1 +fi +grep -q 'MISSING OR UNREADABLE: app/credentials' "$scratch/secrets.log" +# API/rendering errors must not produce an empty reference list and pass. +kubectl() { return 1; } +if check_referenced_secrets >"$scratch/secrets.log"; then + echo 'Secret check accepted a failed manifest render' >&2 + exit 1 +fi +printf '%s\n' 'Deploy validation regressions passed.' diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 2480c13..b9f7ebc 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -88,7 +88,7 @@ jobs: shell: bash run: | set -euo pipefail - tools_dir="$(bash .gitea/workflows/install-ci-tools.sh shellcheck)" + tools_dir="$(bash .gitea/workflows/install-ci-tools.sh shellcheck jq)" export PATH="$tools_dir:$PATH" mapfile -t scripts < <( git ls-files '*.sh' ':(glob)**/*.bash' @@ -98,6 +98,7 @@ jobs: exit 0 fi shellcheck --external-sources --source-path=SCRIPTDIR --severity=style "${scripts[@]}" + bash .gitea/tests/deploy-validation.sh lint-prettier: runs-on: [self-hosted, linux, arch, homelab] diff --git a/.gitea/workflows/compose-lint.sh b/.gitea/workflows/compose-lint.sh index c27e29f..30e79bb 100644 --- a/.gitea/workflows/compose-lint.sh +++ b/.gitea/workflows/compose-lint.sh @@ -21,8 +21,7 @@ # All committed Compose files, including the ones deploy never starts. compose_files() { git ls-files \ - '*/compose.yaml' '*/compose.yml' 'compose.yaml' 'compose.yml' \ - '*/docker-compose.yaml' '*/docker-compose.yml' + '*compose.yaml' '*compose.yml' } # Prints the flags that turn `docker compose config` into the general check. diff --git a/.gitea/workflows/deploy-lib.sh b/.gitea/workflows/deploy-lib.sh index fa7452e..d96880e 100644 --- a/.gitea/workflows/deploy-lib.sh +++ b/.gitea/workflows/deploy-lib.sh @@ -686,27 +686,52 @@ stage_preflight() { git -C "$REPO" reset --hard "$target" } +# Required pod Secrets, scoped to the resource namespace. TLS route Secrets are +# created by cert-manager and are not prerequisites for applying a Certificate. +check_referenced_secrets() { + local m k objects refs extracted ns name + local missing=() + refs="" + for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do + objects="$(kubectl create --dry-run=client --validate=false -f "$m" -o json)" || return 1 + extracted="$(printf '%s' "$objects" | jq -r -f "$REPO/.gitea/workflows/secret-references.jq")" || return 1 + refs+="$extracted"$'\n' + done + for k in ${KUSTOMIZE_APPS[@]+"${KUSTOMIZE_APPS[@]}"}; do + objects="$(kubectl kustomize "$k" | kubectl create --dry-run=client --validate=false -f - -o json)" || return 1 + extracted="$(printf '%s' "$objects" | jq -r -f "$REPO/.gitea/workflows/secret-references.jq")" || return 1 + refs+="$extracted"$'\n' + done + while read -r ns name; do + [ -n "${name:-}" ] || continue + if kubectl get secret "$name" -n "$ns" -o name >/dev/null 2>&1; then + echo " ok: $ns/$name" + else + echo " MISSING OR UNREADABLE: $ns/$name" + missing+=("$ns/$name") + fi + done < <(printf '%s' "$refs" | sort -u) + if [ "${#missing[@]}" -gt 0 ]; then + echo "ERROR: required pod Secrets are missing or unreadable:" + printf ' - %s\n' "${missing[@]}" + echo "Create them in the listed namespaces from the service's secret example." + return 1 + fi +} + stage_validate() { cd "$REPO" select_manifests local m k cf - # Compose .env files and secret files are gitignored by design, so the - # workstation never has real values for the inactive stacks. This stage only - # runs the full check on active stacks; the general structure check for every - # committed Compose file (active or not) lives in the ci workflow, which has no - # .env at all. - # - # Active stacks are still validated with interpolation and env-file resolution - # off, so required-variable guards (:?) and missing local files do not fail the - # deploy. Normalization and consistency checks stay enabled. + # The deploy host has the local .env and secret files. Resolve them here so + # missing configuration fails before either apply job changes workloads. + # CI keeps the structure-only check for inactive stacks. # shellcheck source=compose-lint.sh source "$REPO/.gitea/workflows/compose-lint.sh" - local compose_validate_flags=() - mapfile -t compose_validate_flags < <(compose_safe_flags) log "Validate compose stacks" for cf in ${COMPOSE_STACKS[@]+"${COMPOSE_STACKS[@]}"}; do echo " config: $cf" - validate_compose_file "$cf" ${compose_validate_flags[@]+"${compose_validate_flags[@]}"} + validate_compose_file "$cf" done log "Validate k8s manifests (kubectl dry-run=client)" for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do @@ -724,33 +749,7 @@ stage_validate() { done log "Checking referenced Secrets exist" echo " (deploy never applies *secret*.yaml; create missing ones manually)" - local ref_secrets=() missing_secrets=() all_secrets s - if [ "${#K8S_MANIFESTS[@]}" -gt 0 ]; then - while IFS= read -r s; do - [ -n "$s" ] && ref_secrets+=("$s") - done < <( - { - grep -h -A1 -E 'secretRef:|secretKeyRef:' "${K8S_MANIFESTS[@]}" 2>/dev/null || true - grep -h -E 'secretName:' "${K8S_MANIFESTS[@]}" 2>/dev/null || true - } | grep -E 'name:' | sed -E 's/.*name:[[:space:]]*//' | tr -d '"'"'"' "'"'" | sed -E 's/[[:space:]]*#.*//' | awk 'NF' | sort -u || true - ) - fi - all_secrets="$(kubectl get secrets -A --no-headers -o custom-columns=:metadata.name 2>/dev/null || true)" - for s in ${ref_secrets[@]+"${ref_secrets[@]}"}; do - if printf '%s\n' "$all_secrets" | grep -qx "$s"; then - echo " ok: $s" - else - echo " MISSING: $s" - missing_secrets+=("$s") - fi - done - if [ "${#missing_secrets[@]}" -gt 0 ]; then - echo "ERROR: ${#missing_secrets[@]} referenced Secret(s) not found in the cluster:" - printf ' - %s\n' "${missing_secrets[@]}" - echo "Create them manually from the laptop, e.g.:" - echo " kubectl apply -f SERVICE/k8s/secrets.yaml # see SERVICE/k8s/secrets.yaml.example" - exit 1 - fi + check_referenced_secrets } stage_apply_k8s() { diff --git a/.gitea/workflows/install-ci-tools.sh b/.gitea/workflows/install-ci-tools.sh index 988fe64..2fcb3b3 100755 --- a/.gitea/workflows/install-ci-tools.sh +++ b/.gitea/workflows/install-ci-tools.sh @@ -120,6 +120,15 @@ install_shellcheck() { rm -rf "$tmp" } +install_jq() { + if at_version jq "${JQ_VERSION}"; then + return 0 + fi + fetch "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${goarch}" \ + "$BIN_DIR/jq" + chmod 0755 "$BIN_DIR/jq" +} + install_uv() { if at_version uv "${UV_VERSION}"; then return 0 @@ -236,6 +245,7 @@ for tool in "${wanted[@]}"; do case "$tool" in kubeconform) install_kubeconform ;; shellcheck) install_shellcheck ;; + jq) install_jq ;; actionlint) install_actionlint ;; prettier) install_prettier ;; ruff) install_ruff ;; diff --git a/.gitea/workflows/secret-references.jq b/.gitea/workflows/secret-references.jq new file mode 100644 index 0000000..a205f61 --- /dev/null +++ b/.gitea/workflows/secret-references.jq @@ -0,0 +1,13 @@ +# kubectl emits a List for files containing multiple resources. +(if .kind == "List" then .items[] else . end) +| (.metadata.namespace // "default") as $ns +| [ + (.. | objects + | (.secretRef? // empty), (.secretKeyRef? // empty), (.secret? // empty) + | select(.optional != true) + | .name // .secretName // empty), + (.. | objects | .imagePullSecrets[]?.name) + ] +| unique[] +| select(. != null and . != "") +| "\($ns) \(.)" diff --git a/.gitea/workflows/tool-versions.env b/.gitea/workflows/tool-versions.env index d010495..cf8edb9 100644 --- a/.gitea/workflows/tool-versions.env +++ b/.gitea/workflows/tool-versions.env @@ -31,3 +31,6 @@ UV_VERSION="0.12.17" # so the tree that gets tested is the tree that gets built. Renovate keeps this # in step with the Dockerfile's node: tag via the "node runtime" group. NODE_VERSION="22.23.3" + +# Secret-reference regression tests parse rendered Kubernetes objects. +JQ_VERSION="1.8.1"