`kubectl rollout undo` restores the previous ReplicaSet's pod template
verbatim. While that template names a tag, the rollback does not roll back
the image: the tag has already moved, so the reverted pod pulls the very
build that just failed and the cluster stays broken. The safety net added
in 1505b63 therefore could not recover from a bad image.
Pin the digest at apply time. A digest is not knowable when a manifest is
written, so render_pinned resolves it on the way into the cluster and the
digest is never committed. Git keeps a readable `:prod`, Renovate keeps
seeing exactly the manifests it saw before, and the previous revision of
each workload now holds the digest that was actually serving, so undo
restores those exact bytes.
imagePullPolicy is dropped from the manifests rather than set to
IfNotPresent: a reference that is not `:latest` already defaults to it, and
that is what the Kubernetes docs ask for alongside a digest.
An unresolvable image is fatal instead of a warning, because carrying on
would quietly apply a mutable tag again.
restart_stale_images keeps its comparison but is no longer how a rebuild
reaches the cluster -- the pinned template rolls out on its own now. What
is left is a drift check for hand-run `kubectl set image`, so it matches
the container by repository: a pod's status now reports `repo@sha256:...`
while the manifest still says `:prod`.
The build job stops pushing `:latest` altogether, which removes the tag
that a dev branch could otherwise move under a prod deploy.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
510 lines
17 KiB
YAML
510 lines
17 KiB
YAML
name: ci
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- "**"
|
|
pull_request:
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ci-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
|
|
|
env:
|
|
REGISTRY: gcr.forust.xyz
|
|
|
|
jobs:
|
|
lint-compose:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
# Structure check for every committed Compose file, active or not.
|
|
# Interpolation, env-file and bind-mount resolution are all switched off,
|
|
# because inactive stacks have no .env here and would only fail on their
|
|
# ${VAR:?} guards. Active stacks get the full check with interpolation in
|
|
# the deploy workflow, where the real .env files live.
|
|
- name: Validate Compose files
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
source .gitea/workflows/compose-lint.sh
|
|
|
|
mapfile -t safe_flags < <(compose_safe_flags)
|
|
echo "docker compose config ${safe_flags[*]-}"
|
|
|
|
mapfile -t files < <(compose_files)
|
|
if [ "${#files[@]}" -eq 0 ]; then
|
|
echo "No Compose files found."
|
|
exit 0
|
|
fi
|
|
|
|
failed=0
|
|
for f in "${files[@]}"; do
|
|
if ! out="$(validate_compose_file "$f" ${safe_flags[@]+"${safe_flags[@]}"} 2>&1)"; then
|
|
failed=1
|
|
echo "::error file=${f}::$(printf '%s' "$out" | head -1)"
|
|
fi
|
|
done
|
|
|
|
if [ "$failed" -ne 0 ]; then
|
|
echo "Compose validation failed."
|
|
exit 1
|
|
fi
|
|
echo "checked ${#files[@]} Compose file(s)"
|
|
|
|
lint-actionlint:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
- name: Lint Gitea Actions workflows with actionlint
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
tools_dir="$(bash .gitea/workflows/install-ci-tools.sh actionlint)"
|
|
export PATH="$tools_dir:$PATH"
|
|
actionlint -config-file .gitea/actionlint.yaml -color .gitea/workflows/*.yaml
|
|
|
|
lint-shellcheck:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
- name: Lint shell scripts with ShellCheck
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
tools_dir="$(bash .gitea/workflows/install-ci-tools.sh shellcheck)"
|
|
export PATH="$tools_dir:$PATH"
|
|
# userbot/ is a git subtree synced from forust/userbot, so its shell
|
|
# scripts are upstream's to maintain, not ours. Linting them would let a
|
|
# routine subtree pull turn the deploy gate red on code we do not own.
|
|
mapfile -t scripts < <(
|
|
git ls-files '*.sh' ':(glob)**/*.bash' ':!userbot/**'
|
|
)
|
|
if [ "${#scripts[@]}" -eq 0 ]; then
|
|
echo "No shell scripts found."
|
|
exit 0
|
|
fi
|
|
shellcheck --external-sources --source-path=SCRIPTDIR --severity=style "${scripts[@]}"
|
|
|
|
lint-prettier:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
- name: Check formatting with Prettier
|
|
shell: bash
|
|
run: |
|
|
mapfile -t prettier_files < <(
|
|
git ls-files \
|
|
| grep -E '\.(md|json|ya?ml|html|css)$' \
|
|
| grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
|
|
)
|
|
|
|
if [ "${#prettier_files[@]}" -eq 0 ]; then
|
|
echo "No Prettier-managed files found."
|
|
exit 0
|
|
fi
|
|
|
|
prettier --check --ignore-unknown "${prettier_files[@]}"
|
|
|
|
lint-ruff:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
- name: Lint Python with Ruff
|
|
shell: bash
|
|
run: |
|
|
ruff check .
|
|
|
|
lint-yaml:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
- name: Lint YAML syntax
|
|
shell: bash
|
|
run: |
|
|
mapfile -t yaml_files < <(
|
|
git ls-files '*.yaml' '*.yml' \
|
|
':!node_modules/**' \
|
|
':!**/.venv/**'
|
|
)
|
|
|
|
if [ "${#yaml_files[@]}" -eq 0 ]; then
|
|
echo "No YAML files found."
|
|
exit 0
|
|
fi
|
|
|
|
yamllint -c .yamllint "${yaml_files[@]}"
|
|
|
|
lint-dockerfiles:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
- name: Lint Dockerfiles
|
|
shell: bash
|
|
run: |
|
|
mapfile -t dockerfiles < <(
|
|
git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
|
|
)
|
|
|
|
if [ "${#dockerfiles[@]}" -eq 0 ]; then
|
|
echo "No Dockerfiles found."
|
|
exit 0
|
|
fi
|
|
|
|
hadolint -c .hadolint.yaml "${dockerfiles[@]}"
|
|
|
|
validate:
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 20
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
|
|
- name: Validate Kubernetes manifests against JSON schemas
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
|
|
export PATH="$tools_dir:$PATH"
|
|
|
|
mapfile -t manifests < <(
|
|
git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
|
|
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
|
|
)
|
|
|
|
if [ "${#manifests[@]}" -eq 0 ]; then
|
|
echo "No Kubernetes manifests found."
|
|
exit 0
|
|
fi
|
|
|
|
kubeconform \
|
|
-strict \
|
|
-ignore-missing-schemas \
|
|
-summary \
|
|
"${manifests[@]}"
|
|
|
|
# kubeconform has no schemas for CRDs, so every IngressRoute, Certificate,
|
|
# PrometheusRule, Middleware, ServersTransport and ServiceMonitor is silently
|
|
# skipped above. The live API server knows the real CRD schemas (and runs the
|
|
# cert-manager / Traefik admission webhooks), so validate there too.
|
|
#
|
|
# Only services marked with a k8s/active marker are checked: server-side
|
|
# dry-run needs the target namespace to exist, and inactive services are not
|
|
# deployed. Services being enabled for the first time are still covered by
|
|
# the JSON-schema pass above.
|
|
- name: Validate active manifests against the live API server
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
if ! kubectl get --raw='/readyz' --request-timeout=10s >/dev/null 2>&1; then
|
|
echo "::warning::Cluster unreachable — skipped server-side validation of CRDs (IngressRoute, Certificate, PrometheusRule). Review manifest changes manually."
|
|
exit 0
|
|
fi
|
|
|
|
mapfile -t k8s_dirs < <(
|
|
git ls-files '*.yaml' '*.yml' \
|
|
| grep -E '(^|/)k8s/' \
|
|
| sed -E 's#((^|.*/)k8s)/.*#\1#' \
|
|
| sort -u
|
|
)
|
|
|
|
manifests=()
|
|
kustomize_apps=()
|
|
for dir in "${k8s_dirs[@]}"; do
|
|
if [ ! -f "${dir}/active" ]; then
|
|
echo "skip (no k8s/active): ${dir}"
|
|
continue
|
|
fi
|
|
if [ -f "${dir}/overlays/prod/kustomization.yaml" ]; then
|
|
kustomize_apps+=("${dir}/overlays/prod")
|
|
elif [ -f "${dir}/base/kustomization.yaml" ]; then
|
|
kustomize_apps+=("${dir}/base")
|
|
else
|
|
while IFS= read -r f; do
|
|
[ -n "$f" ] && manifests+=("$f")
|
|
done < <(
|
|
git ls-files "${dir}/*.yaml" "${dir}/*.yml" \
|
|
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
|
|
)
|
|
fi
|
|
done
|
|
|
|
echo "server-side dry-run: ${#manifests[@]} manifests, ${#kustomize_apps[@]} kustomize apps"
|
|
failed=0
|
|
for m in ${manifests[@]+"${manifests[@]}"}; do
|
|
if ! out="$(kubectl apply --dry-run=server -f "$m" 2>&1)"; then
|
|
failed=1
|
|
echo "::error file=${m}::$(printf '%s' "$out" | head -1)"
|
|
fi
|
|
done
|
|
for k in ${kustomize_apps[@]+"${kustomize_apps[@]}"}; do
|
|
if ! out="$(kubectl apply -k "$k" --dry-run=server 2>&1)"; then
|
|
failed=1
|
|
echo "::error file=${k}::$(printf '%s' "$out" | head -1)"
|
|
fi
|
|
done
|
|
|
|
if [ "$failed" -ne 0 ]; then
|
|
echo "Server-side validation failed. The API server (or an admission webhook) rejected these manifests."
|
|
exit 1
|
|
fi
|
|
echo "server-side dry-run: all active manifests accepted by the API server"
|
|
|
|
build:
|
|
needs:
|
|
[lint-actionlint, lint-shellcheck, lint-compose, lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
|
|
if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
|
|
runs-on: [self-hosted, linux, arch, homelab]
|
|
timeout-minutes: 60
|
|
outputs:
|
|
services: ${{ steps.services.outputs.services }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Detect changed docker-built services
|
|
id: services
|
|
shell: bash
|
|
run: |
|
|
base="${{ github.event.before }}"
|
|
if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
|
|
base="$(git rev-list --max-parents=0 HEAD)"
|
|
fi
|
|
|
|
mapfile -t changed_files < <(git diff --name-only "$base" "${GITHUB_SHA}")
|
|
|
|
services=()
|
|
|
|
add_service() {
|
|
local name="$1"
|
|
local seen=0
|
|
for existing in "${services[@]}"; do
|
|
if [ "$existing" = "$name" ]; then
|
|
seen=1
|
|
break
|
|
fi
|
|
done
|
|
if [ "$seen" -eq 0 ]; then
|
|
services+=("$name")
|
|
fi
|
|
}
|
|
|
|
for file in "${changed_files[@]}"; do
|
|
case "$file" in
|
|
dtek_notif/*)
|
|
add_service dtek_notif
|
|
;;
|
|
errorpages/*)
|
|
add_service errorpages
|
|
;;
|
|
userbot/*)
|
|
add_service userbot
|
|
;;
|
|
homepages/*)
|
|
add_service homepages
|
|
;;
|
|
edu_master/phpsessid-bot/*|edu_master/webinar-checker/*|edu_master/compose.yaml)
|
|
add_service edu_master
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ "${#services[@]}" -eq 0 ]; then
|
|
echo "No docker-built services changed."
|
|
echo "services=" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
|
|
printf '%s\n' "${services[@]}" | tee /tmp/services.txt
|
|
echo "services=$(paste -sd, /tmp/services.txt)" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Log in to registry
|
|
if: steps.services.outputs.services != ''
|
|
shell: bash
|
|
run: |
|
|
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
|
|
-u "${{ secrets.REGISTRY_USERNAME }}" \
|
|
--password-stdin
|
|
|
|
- name: Build and push changed images
|
|
if: steps.services.outputs.services != ''
|
|
shell: bash
|
|
run: |
|
|
IFS=, read -r -a services <<< "${{ steps.services.outputs.services }}"
|
|
|
|
for service in "${services[@]}"; do
|
|
case "$service" in
|
|
dtek_notif)
|
|
image="${REGISTRY}/forust/dtek-notif"
|
|
tags=()
|
|
case "${GITHUB_REF_NAME}" in
|
|
main)
|
|
tags+=("main" "prod")
|
|
;;
|
|
dev)
|
|
tags+=("dev")
|
|
;;
|
|
esac
|
|
build_args=()
|
|
for tag in "${tags[@]}"; do
|
|
build_args+=(-t "${image}:${tag}")
|
|
done
|
|
docker build \
|
|
--cache-from "type=registry,ref=${image}:buildcache" \
|
|
--cache-to "type=registry,ref=${image}:buildcache,mode=max" \
|
|
"${build_args[@]}" dtek_notif
|
|
for tag in "${tags[@]}"; do
|
|
docker push "${image}:${tag}"
|
|
done
|
|
;;
|
|
errorpages)
|
|
image="${REGISTRY}/forust/error-pages"
|
|
tags=()
|
|
case "${GITHUB_REF_NAME}" in
|
|
main)
|
|
tags+=("main" "prod")
|
|
;;
|
|
dev)
|
|
tags+=("dev")
|
|
;;
|
|
esac
|
|
build_args=()
|
|
for tag in "${tags[@]}"; do
|
|
build_args+=(-t "${image}:${tag}")
|
|
done
|
|
docker build \
|
|
--cache-from "type=registry,ref=${image}:buildcache" \
|
|
--cache-to "type=registry,ref=${image}:buildcache,mode=max" \
|
|
"${build_args[@]}" errorpages
|
|
for tag in "${tags[@]}"; do
|
|
docker push "${image}:${tag}"
|
|
done
|
|
;;
|
|
userbot)
|
|
tags=()
|
|
case "${GITHUB_REF_NAME}" in
|
|
main)
|
|
tags+=("main" "prod")
|
|
;;
|
|
dev)
|
|
tags+=("dev")
|
|
;;
|
|
esac
|
|
for target in runtime panel; do
|
|
case "$target" in
|
|
runtime)
|
|
context="userbot"
|
|
image="${REGISTRY}/forust/userbot"
|
|
;;
|
|
panel)
|
|
context="userbot/panel"
|
|
image="${REGISTRY}/forust/userbot-panel"
|
|
;;
|
|
esac
|
|
build_args=()
|
|
for tag in "${tags[@]}"; do
|
|
build_args+=(-t "${image}:${tag}")
|
|
done
|
|
docker build \
|
|
--cache-from "type=registry,ref=${image}:buildcache" \
|
|
--cache-to "type=registry,ref=${image}:buildcache,mode=max" \
|
|
"${build_args[@]}" "$context"
|
|
for tag in "${tags[@]}"; do
|
|
docker push "${image}:${tag}"
|
|
done
|
|
done
|
|
;;
|
|
homepages)
|
|
for variant in forust xdfnx; do
|
|
case "$variant" in
|
|
forust)
|
|
image="${REGISTRY}/forust/forust-homepage"
|
|
;;
|
|
xdfnx)
|
|
image="${REGISTRY}/forust/xdfnx-homepage"
|
|
;;
|
|
esac
|
|
tags=()
|
|
case "${GITHUB_REF_NAME}" in
|
|
main)
|
|
tags+=("main" "prod")
|
|
;;
|
|
dev)
|
|
tags+=("dev")
|
|
;;
|
|
esac
|
|
build_args=()
|
|
for tag in "${tags[@]}"; do
|
|
build_args+=(-t "${image}:${tag}")
|
|
done
|
|
docker build \
|
|
--cache-from "type=registry,ref=${image}:buildcache" \
|
|
--cache-to "type=registry,ref=${image}:buildcache,mode=max" \
|
|
"${build_args[@]}" -f "homepages/Dockerfile.${variant}" homepages
|
|
for tag in "${tags[@]}"; do
|
|
docker push "${image}:${tag}"
|
|
done
|
|
done
|
|
;;
|
|
edu_master)
|
|
for variant in session-keeper webinar-checker; do
|
|
case "$variant" in
|
|
session-keeper)
|
|
context="edu_master/phpsessid-bot"
|
|
image="${REGISTRY}/forust/session-keeper"
|
|
;;
|
|
webinar-checker)
|
|
context="edu_master/webinar-checker"
|
|
image="${REGISTRY}/forust/webinar-checker"
|
|
;;
|
|
esac
|
|
tags=()
|
|
case "${GITHUB_REF_NAME}" in
|
|
main)
|
|
tags+=("main" "prod")
|
|
;;
|
|
dev)
|
|
tags+=("dev")
|
|
;;
|
|
esac
|
|
build_args=()
|
|
for tag in "${tags[@]}"; do
|
|
build_args+=(-t "${image}:${tag}")
|
|
done
|
|
docker build \
|
|
--cache-from "type=registry,ref=${image}:buildcache" \
|
|
--cache-to "type=registry,ref=${image}:buildcache,mode=max" \
|
|
"${build_args[@]}" "$context"
|
|
for tag in "${tags[@]}"; do
|
|
docker push "${image}:${tag}"
|
|
done
|
|
done
|
|
;;
|
|
esac
|
|
done
|