351 lines
14 KiB
Python
351 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""CI release artifacts and the SHA-specific Gitea deployment gate (stdlib only)."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import itertools
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
SHA = re.compile(r'[0-9a-f]{40}')
|
|
DIGEST = re.compile(r'sha256:[0-9a-f]{64}')
|
|
IMAGES = {
|
|
'error-pages': ('errorpages', 'errorpages/Dockerfile'),
|
|
'forust-homepage': ('homepages', 'homepages/Dockerfile.forust'),
|
|
'xdfnx-homepage': ('homepages', 'homepages/Dockerfile.xdfnx'),
|
|
}
|
|
|
|
# These images are released by the EDU application repository.
|
|
EXTERNAL_IMAGES = {'gcr.forust.xyz/forust/session-keeper', 'gcr.forust.xyz/forust/webinar-checker'}
|
|
|
|
|
|
def command(*args, **kwargs):
|
|
"""Arguments are passed directly to the executable, never to a shell."""
|
|
return subprocess.check_output(args, text=True, **kwargs).strip() # noqa: S603, S607
|
|
|
|
|
|
def validate_release(data, sha=None):
|
|
if data.get('version') != 1 or not SHA.fullmatch(data.get('sha', '')):
|
|
raise ValueError('Invalid release version or SHA')
|
|
if sha is not None and data['sha'] != sha:
|
|
raise ValueError('Release SHA does not match the checked CI commit')
|
|
expected = {f'gcr.forust.xyz/forust/{name}' for name in IMAGES}
|
|
if set(data.get('images', {})) != expected:
|
|
raise ValueError('Release must contain all owned images')
|
|
if not all(DIGEST.fullmatch(value) for value in data['images'].values()):
|
|
raise ValueError('Release has an invalid image digest')
|
|
if set(data.get('inputs', {})) != expected or not all(
|
|
re.fullmatch(r'[0-9a-f]{64}', value) for value in data['inputs'].values()
|
|
):
|
|
raise ValueError('Release has invalid build input fingerprints')
|
|
return data
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, _req, _fp, _code, _msg, _headers, _newurl):
|
|
return None
|
|
|
|
|
|
class Gitea:
|
|
def __init__(self):
|
|
self.origin = os.environ['GITHUB_SERVER_URL'].rstrip('/')
|
|
if urllib.parse.urlsplit(self.origin).scheme != 'https':
|
|
raise ValueError('Gitea API must use HTTPS')
|
|
self.repository = os.environ['GITHUB_REPOSITORY']
|
|
if not re.fullmatch(r'[\w.-]+/[\w.-]+', self.repository):
|
|
raise ValueError('Invalid Gitea repository')
|
|
self.token = os.environ['GITEA_TOKEN']
|
|
self.base = f'{self.origin}/api/v1/repos/{self.repository}'
|
|
|
|
def request(self, url, *, archive=False):
|
|
if not url.startswith(self.base + '/'):
|
|
raise ValueError('Refusing to send the Actions token to another origin')
|
|
req = urllib.request.Request(url, headers={'Authorization': f'token {self.token}'}) # noqa: S310 -- HTTPS origin validated above
|
|
opener = urllib.request.build_opener(NoRedirect())
|
|
try:
|
|
response = opener.open(req, timeout=30) # noqa: S310
|
|
except urllib.error.HTTPError as error:
|
|
if not archive or error.code not in (301, 302, 303, 307, 308):
|
|
raise RuntimeError(f'Gitea API returned HTTP {error.code}') from None
|
|
target = urllib.parse.urljoin(url, error.headers['Location'])
|
|
if urllib.parse.urlsplit(target).scheme != 'https':
|
|
raise ValueError('Artifact redirect must use HTTPS') from None
|
|
# Signed storage redirects must never receive the Gitea token.
|
|
response = urllib.request.urlopen(target, timeout=30) # noqa: S310
|
|
with response:
|
|
payload = response.read(8 * 1024 * 1024 + 1)
|
|
if len(payload) > 8 * 1024 * 1024:
|
|
raise ValueError('Gitea response exceeds 8 MiB')
|
|
return payload if archive else json.loads(payload)
|
|
|
|
def pages(self, path, key, **params):
|
|
for page in range(1, 101):
|
|
query = urllib.parse.urlencode({**params, 'page': page, 'limit': 50})
|
|
data = self.request(f'{self.base}/{path}?{query}')
|
|
entries = data[key]
|
|
yield from entries
|
|
if len(entries) < 50:
|
|
return
|
|
raise RuntimeError('Gitea pagination limit exceeded')
|
|
|
|
def successful_runs(self, sha=None):
|
|
params = {'branch': 'main', 'status': 'success', 'exclude_pull_requests': 'true'}
|
|
if sha:
|
|
params['head_sha'] = sha
|
|
for run in self.pages('actions/workflows/ci.yaml/runs', 'workflow_runs', **params):
|
|
if (
|
|
run.get('status') == 'completed'
|
|
and run.get('conclusion') == 'success'
|
|
and run.get('head_branch') == 'main'
|
|
and run.get('event') in ('push', 'workflow_dispatch')
|
|
and (run.get('repository') or {}).get('full_name') == self.repository
|
|
and (run.get('head_repository') or run.get('repository') or {}).get('full_name') == self.repository
|
|
and (sha is None or run.get('head_sha') == sha)
|
|
):
|
|
yield run
|
|
|
|
def release(self, run):
|
|
sha = run['head_sha']
|
|
jobs = list(self.pages(f'actions/runs/{run["id"]}/jobs', 'jobs'))
|
|
# A green workflow with a skipped build must not authorize a deploy.
|
|
if not any(job.get('name') == 'build' and job.get('conclusion') == 'success' for job in jobs):
|
|
raise ValueError('CI build job did not succeed')
|
|
artifacts = self.request(f'{self.base}/actions/runs/{run["id"]}/artifacts')['artifacts']
|
|
matching = [a for a in artifacts if a['name'] == f'release-{sha}' and not a.get('expired')]
|
|
if len(matching) != 1:
|
|
raise ValueError('CI release artifact is missing, expired or ambiguous; rerun CI')
|
|
blob = self.request(f'{self.base}/actions/artifacts/{matching[0]["id"]}/zip', archive=True)
|
|
with zipfile.ZipFile(io.BytesIO(blob)) as archive:
|
|
files = [entry for entry in archive.infolist() if not entry.is_dir()]
|
|
if len(files) != 1 or files[0].filename != 'release.json' or files[0].file_size > 256 * 1024:
|
|
raise ValueError('Unexpected release archive contents')
|
|
return validate_release(json.loads(archive.read(files[0])), sha)
|
|
|
|
|
|
def fingerprint(context, dockerfile):
|
|
tree = command('git', 'ls-tree', '-r', 'HEAD', '--', context, dockerfile, '.gitea/workflows/release.py')
|
|
return hashlib.sha256(tree.encode()).hexdigest()
|
|
|
|
|
|
def gate(output, requested_ref, event_sha):
|
|
command('git', 'fetch', '--quiet', 'origin', 'main')
|
|
if event_sha:
|
|
if not SHA.fullmatch(event_sha):
|
|
raise ValueError('Invalid workflow_run SHA')
|
|
sha = event_sha
|
|
else:
|
|
if requested_ref == 'main':
|
|
requested_ref = 'origin/main'
|
|
sha = command('git', 'rev-parse', '--verify', '--end-of-options', f'{requested_ref}^{{commit}}')
|
|
if not SHA.fullmatch(sha):
|
|
raise ValueError('Invalid deploy SHA')
|
|
command('git', 'merge-base', '--is-ancestor', sha, 'origin/main')
|
|
api = Gitea()
|
|
runs = list(api.successful_runs(sha))
|
|
if not runs:
|
|
raise ValueError(f'No successful main CI for {sha}; run CI before deploying')
|
|
release = api.release(max(runs, key=lambda run: run['id']))
|
|
output.write_text(json.dumps(release, indent=2) + '\n')
|
|
if os.environ.get('GITHUB_OUTPUT'):
|
|
with Path(os.environ['GITHUB_OUTPUT']).open('a') as stream:
|
|
stream.write(f'sha={sha}\n')
|
|
print(f'CI gate accepted {sha}')
|
|
|
|
|
|
def build(output):
|
|
sha = command('git', 'rev-parse', 'HEAD')
|
|
if sha != os.environ['GITHUB_SHA'] or not SHA.fullmatch(sha):
|
|
raise ValueError('Build checkout does not match GITHUB_SHA')
|
|
api = Gitea()
|
|
previous = None
|
|
for run in sorted(itertools.islice(api.successful_runs(), 50), key=lambda item: item['id'], reverse=True):
|
|
if str(run['id']) == os.environ.get('GITHUB_RUN_ID'):
|
|
continue
|
|
try:
|
|
previous = api.release(run)
|
|
break
|
|
except ValueError:
|
|
# Expired artifacts only cost a rebuild; mutable tags are never a fallback.
|
|
continue
|
|
docker_config = tempfile.mkdtemp(prefix='homelab-registry-')
|
|
builder_config = Path.home() / '.cache/homelab-ci/buildx'
|
|
builder_config.mkdir(parents=True, exist_ok=True)
|
|
env = {**os.environ, 'DOCKER_CONFIG': docker_config, 'BUILDX_CONFIG': str(builder_config)}
|
|
try:
|
|
subprocess.run( # noqa: S603, S607
|
|
[
|
|
shutil.which('docker') or '/usr/bin/docker',
|
|
'login',
|
|
'gcr.forust.xyz',
|
|
'-u',
|
|
os.environ['REGISTRY_USERNAME'],
|
|
'--password-stdin',
|
|
],
|
|
input=os.environ['REGISTRY_PASSWORD'],
|
|
text=True,
|
|
check=True,
|
|
env=env,
|
|
)
|
|
builder = 'homelab-ci'
|
|
versions = dict(
|
|
re.findall(r'^([A-Z_]+)="([^"\n]+)"$', Path('.gitea/workflows/tool-versions.env').read_text(), re.MULTILINE)
|
|
)
|
|
image = versions['BUILDKIT_IMAGE']
|
|
signature = builder_config / 'homelab-ci-image'
|
|
exists = (
|
|
subprocess.run( # noqa: S603
|
|
[shutil.which('docker') or '/usr/bin/docker', 'buildx', 'inspect', builder],
|
|
capture_output=True,
|
|
env=env,
|
|
).returncode
|
|
== 0
|
|
)
|
|
if exists and (not signature.exists() or signature.read_text().strip() != image):
|
|
command('docker', 'buildx', 'rm', '--keep-state', builder, env=env)
|
|
exists = False
|
|
if not exists:
|
|
command(
|
|
'docker',
|
|
'buildx',
|
|
'create',
|
|
'--name',
|
|
builder,
|
|
'--driver',
|
|
'docker-container',
|
|
'--driver-opt',
|
|
f'image={image}',
|
|
'--buildkitd-config',
|
|
'.gitea/runner/buildkitd.toml',
|
|
env=env,
|
|
)
|
|
signature.write_text(image + '\n')
|
|
release = {'version': 1, 'sha': sha, 'images': {}, 'inputs': {}}
|
|
for name, (context, dockerfile) in IMAGES.items():
|
|
image = f'gcr.forust.xyz/forust/{name}'
|
|
inputs = fingerprint(context, dockerfile)
|
|
old_digest = (previous or {}).get('images', {}).get(image)
|
|
exists = False
|
|
if old_digest and previous['inputs'].get(image) == inputs:
|
|
exists = (
|
|
subprocess.run( # noqa: S603, S607
|
|
[
|
|
shutil.which('docker') or '/usr/bin/docker',
|
|
'buildx',
|
|
'imagetools',
|
|
'inspect',
|
|
f'{image}@{old_digest}',
|
|
],
|
|
capture_output=True,
|
|
env=env,
|
|
timeout=60,
|
|
).returncode
|
|
== 0
|
|
)
|
|
if exists:
|
|
print(f'Reuse {name}: inputs unchanged')
|
|
digest = old_digest
|
|
else:
|
|
print(f'Build {name}', flush=True)
|
|
metadata = Path(docker_config) / 'metadata.json'
|
|
command(
|
|
'docker',
|
|
'buildx',
|
|
'build',
|
|
'--builder',
|
|
builder,
|
|
'--push',
|
|
'--platform',
|
|
'linux/amd64',
|
|
'--provenance=false',
|
|
'--cache-from',
|
|
f'type=registry,ref={image}:buildcache',
|
|
'--cache-to',
|
|
f'type=registry,ref={image}:buildcache,mode=max',
|
|
'--tag',
|
|
f'{image}:sha-{sha}',
|
|
'--metadata-file',
|
|
str(metadata),
|
|
'--file',
|
|
dockerfile,
|
|
context,
|
|
env=env,
|
|
)
|
|
digest = json.loads(metadata.read_text())['containerimage.digest']
|
|
release['images'][image] = digest
|
|
release['inputs'][image] = inputs
|
|
validate_release(release, sha)
|
|
output.write_text(json.dumps(release, indent=2) + '\n')
|
|
finally:
|
|
# Cleanup errors must neither leak credentials nor mask the original build error.
|
|
try:
|
|
subprocess.run( # noqa: S603
|
|
[
|
|
shutil.which('docker') or '/usr/bin/docker',
|
|
'buildx',
|
|
'prune',
|
|
'--builder',
|
|
'homelab-ci',
|
|
'--force',
|
|
'--max-used-space',
|
|
'1gb',
|
|
],
|
|
env=env,
|
|
timeout=60,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
print('CI builder cache cleanup deferred', flush=True)
|
|
finally:
|
|
shutil.rmtree(docker_config)
|
|
|
|
|
|
def render(stream, destination):
|
|
release = validate_release(json.loads(Path(os.environ['RELEASE_FILE']).read_text()), os.environ['DEPLOY_SHA'])
|
|
image_line = re.compile(
|
|
r"^(\s*(?:-\s*)?image:\s*)(['\"]?)(gcr\.forust\.xyz/forust/[\w.-]+)(?::[\w.-]+|@sha256:[0-9a-f]{64})\2(\s*(?:#.*)?)$"
|
|
)
|
|
rendered = []
|
|
for line in stream:
|
|
match = image_line.fullmatch(line.rstrip('\n'))
|
|
if match:
|
|
prefix, quote, image, tail = match.groups()
|
|
if image in EXTERNAL_IMAGES and f'{image}@sha256:' in line:
|
|
rendered.append(line)
|
|
continue
|
|
if image not in release['images']:
|
|
raise ValueError(f'Owned image missing from checked release: {image}')
|
|
line = f'{prefix}{quote}{image}@{release["images"][image]}{quote}{tail}\n'
|
|
elif re.match(r'\s*(?:-\s*)?image:', line) and 'gcr.forust.xyz/forust/' in line:
|
|
raise ValueError('Unsupported owned image syntax; refusing to apply a mutable tag')
|
|
rendered.append(line)
|
|
destination.writelines(rendered)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('build', 'gate', 'render'))
|
|
parser.add_argument('--output', type=Path, default=Path('release.json'))
|
|
parser.add_argument('--ref', default='main')
|
|
parser.add_argument('--event-sha', default='')
|
|
args = parser.parse_args()
|
|
if args.action == 'render':
|
|
render(sys.stdin, sys.stdout)
|
|
elif args.action == 'gate':
|
|
gate(args.output, args.ref, args.event_sha)
|
|
else:
|
|
build(args.output)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|