fix(ci): validate image digests and isolate test outputs

tests/ci_test_case.py

@@ -6,6 +6,8 @@ import unittest
 from pathlib import Path
 from unittest.mock import patch
 
+CI_COMMAND_FILES = ('GITHUB_STEP_SUMMARY', 'GITHUB_OUTPUT', 'GITHUB_ENV', 'GITHUB_PATH', 'GITHUB_STATE')
+
 
 class IsolatedCITestCase(unittest.TestCase):
     def setUp(self):
@@ -13,7 +15,7 @@ class IsolatedCITestCase(unittest.TestCase):
         directory = tempfile.TemporaryDirectory(prefix='homelab-test-ci-')
         self.addCleanup(directory.cleanup)
         paths = {}
-        for variable in ('GITHUB_STEP_SUMMARY', 'GITHUB_OUTPUT'):
+        for variable in CI_COMMAND_FILES:
             path = Path(directory.name) / variable
             path.touch()
             paths[variable] = str(path)

tests/test_ci_output_isolation.py

@@ -5,11 +5,34 @@ import subprocess
 import sys
 import tempfile
 from pathlib import Path
+from unittest.mock import patch
 
-from ci_test_case import IsolatedCITestCase
+from ci_test_case import CI_COMMAND_FILES, IsolatedCITestCase
 
 
 class CIOutputIsolationTests(IsolatedCITestCase):
+    def test_all_command_files_are_private_and_environment_is_restored(self):
+        with tempfile.TemporaryDirectory() as scratch:
+            external = {variable: str(Path(scratch) / variable) for variable in CI_COMMAND_FILES}
+            for path in external.values():
+                Path(path).write_text('external CI file\n')
+            with patch.dict(os.environ, external):
+                probe = IsolatedCITestCase()
+                probe.setUp()
+                private = []
+                try:
+                    for variable in CI_COMMAND_FILES:
+                        self.assertNotEqual(os.environ[variable], external[variable])
+                        path = Path(os.environ[variable])
+                        private.append(path)
+                        path.write_text('test-only command\n')
+                finally:
+                    probe.doCleanups()
+                for variable in CI_COMMAND_FILES:
+                    self.assertEqual(os.environ[variable], external[variable])
+                    self.assertEqual(Path(external[variable]).read_text(), 'external CI file\n')
+                self.assertTrue(all(not path.exists() for path in private))
+
     def test_unit_suite_preserves_external_ci_files(self):
         tests = Path(__file__).resolve().parent
         modules = sorted(p.stem for p in tests.glob('test_*.py') if p.name != Path(__file__).name)
@@ -17,7 +40,7 @@ class CIOutputIsolationTests(IsolatedCITestCase):
             environment = os.environ.copy()
             environment['PYTHONPATH'] = str(tests) + os.pathsep + environment.get('PYTHONPATH', '')
             expected = {}
-            for variable in ('GITHUB_STEP_SUMMARY', 'GITHUB_OUTPUT'):
+            for variable in CI_COMMAND_FILES:
                 path = Path(scratch) / variable
                 content = f'external {variable}\n'
                 path.write_text(content)

tests/test_cicd_lifecycle.py

@@ -226,6 +226,77 @@ class FailureSummaryTests(IsolatedCITestCase):
             self.assertIn('xdfnx-homepage', content)
             self.assertNotIn('private value', content)
 
+    def test_invalid_digest_is_not_reported_as_a_completed_image(self):
+        for digest in ('invalid-private-metadata', None, ['invalid']):
+            with self.subTest(digest=digest), tempfile.TemporaryDirectory() as scratch:
+                root = Path(scratch)
+                summary = root / 'summary.md'
+                name = 'error-pages'
+                context, dockerfile = release_module.IMAGES[name]
+                plan = {
+                    'sha': 'a' * 40,
+                    'targets': [
+                        {
+                            'name': name,
+                            'context': context,
+                            'dockerfile': dockerfile,
+                            'inputs': 'c' * 64,
+                            'reuse_digest': None,
+                        }
+                    ],
+                }
+
+                def fake_command(*args, digest=digest, **_kwargs):
+                    if args[:3] == ('docker', 'buildx', 'build'):
+                        Path(args[args.index('--metadata-file') + 1]).write_text(
+                            json.dumps({'containerimage.digest': digest})
+                        )
+                    return ''
+
+                with (
+                    patch.dict(
+                        os.environ,
+                        {
+                            'GITHUB_STEP_SUMMARY': str(summary),
+                            'GITHUB_SHA': 'a' * 40,
+                            'REGISTRY_USERNAME': 'test',
+                            'REGISTRY_PASSWORD': 'placeholder',
+                        },
+                    ),
+                    patch.object(release_module, 'checked_plan', return_value=plan),
+                    patch.object(release_module.Path, 'home', return_value=root),
+                    patch.object(release_module, 'command', side_effect=fake_command),
+                    patch.object(subprocess, 'run', return_value=subprocess.CompletedProcess([], 0)),
+                    self.assertRaisesRegex(ValueError, 'invalid digest'),
+                ):
+                    release_module.build(root / 'image.json', name, root / 'plan.json')
+                self.assertFalse((root / 'image.json').exists())
+                content = summary.read_text()
+                self.assertIn('**failure**', content)
+                self.assertIn('### Built\n- None', content)
+                self.assertIn('### Completed image digests\n- None', content)
+                self.assertNotIn('invalid-private-metadata', content)
+
+    def test_successful_image_result_does_not_claim_complete_release(self):
+        def complete_image(_output, report, _name, _plan):
+            report.update(phase='Image result file saved', built=['error-pages'])
+            report['images']['gcr.forust.xyz/forust/error-pages'] = 'sha256:' + 'b' * 64
+
+        with (
+            patch.dict(os.environ, {'GITHUB_SHA': 'a' * 40}),
+            patch.object(
+                release_module,
+                'build_images',
+                side_effect=complete_image,
+            ),
+        ):
+            release_module.build(Path('unused.json'), 'error-pages', Path('unused-plan.json'))
+        content = Path(os.environ['GITHUB_STEP_SUMMARY']).read_text()
+        self.assertIn('## Image build result `error-pages`', content)
+        self.assertIn('Commit: `' + 'a' * 40 + '`', content)
+        self.assertIn('final build job must publish the complete release', content)
+        self.assertNotIn('## Image release', content)
+
     def test_deploy_failure_reports_completed_apply_and_rollback_result(self):
         with tempfile.TemporaryDirectory() as scratch:
             state = Path(scratch)