forustandClaude Opus 4.8
0ae0df7473
ruff.toml has declared `quote-style = "single"` and line-length 120 since the lint job landed, and 118 of 128 files follow it. The panel backend and the netbox configuration were written in black/prettier style instead, so a `ruff format --check` would have failed on them from the start. Bring them onto the style the repository already declares, which is what makes the check adoptable at all. Formatting only: apart from quote style the diff is multi-line expressions joined where they fit inside 120 columns. Both suites still pass afterwards (25 pytest, and ruff check is clean). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
537 lines
21 KiB
Python
537 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from threading import Lock
|
|
from typing import Any
|
|
|
|
from kubernetes import client, config
|
|
from kubernetes.client.exceptions import ApiException
|
|
|
|
from .config import Settings
|
|
from .errors import PanelError
|
|
from .models import AccountBase, InstanceSummary
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
MANAGED_LABEL = 'app.kubernetes.io/name=userbot'
|
|
INSTANCE_LABEL = 'app.kubernetes.io/instance'
|
|
MANAGED_BY_LABEL = 'app.kubernetes.io/managed-by'
|
|
DISPLAY_ANNOTATION = 'userbot.forust.xyz/display-name'
|
|
LEGACY_ANNOTATION = 'userbot.forust.xyz/legacy'
|
|
CREDENTIALS_ANNOTATION = 'userbot.forust.xyz/credentials-secret'
|
|
PVC_ANNOTATION = 'userbot.forust.xyz/pvc'
|
|
RESTART_ANNOTATION = 'userbot.forust.xyz/restarted-at'
|
|
|
|
|
|
def _selector(labels: dict[str, str] | None) -> str:
|
|
return ','.join(f'{key}={value}' for key, value in (labels or {}).items())
|
|
|
|
|
|
def _as_datetime(value: Any) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, datetime):
|
|
return value
|
|
return getattr(value, 'replace', lambda **_: None)(tzinfo=UTC)
|
|
|
|
|
|
class KubernetesService:
|
|
def __init__(
|
|
self,
|
|
settings: Settings,
|
|
*,
|
|
core: client.CoreV1Api | None = None,
|
|
apps: client.AppsV1Api | None = None,
|
|
custom: client.CustomObjectsApi | None = None,
|
|
) -> None:
|
|
self.settings = settings
|
|
if core is None or apps is None:
|
|
try:
|
|
config.load_incluster_config()
|
|
except config.ConfigException:
|
|
try:
|
|
config.load_kube_config()
|
|
except Exception as exc:
|
|
raise PanelError(
|
|
503,
|
|
'No in-cluster or kubeconfig configuration is available',
|
|
) from exc
|
|
self.core = core or client.CoreV1Api()
|
|
self.apps = apps or client.AppsV1Api()
|
|
self.custom = custom or client.CustomObjectsApi()
|
|
self._provision_lock = Lock()
|
|
|
|
def ensure_prerequisites(self) -> None:
|
|
try:
|
|
self.core.read_namespaced_secret(
|
|
self.settings.common_secret,
|
|
self.settings.namespace,
|
|
)
|
|
self.core.read_namespaced_config_map(
|
|
self.settings.common_config,
|
|
self.settings.namespace,
|
|
)
|
|
except ApiException as exc:
|
|
if exc.status == 404:
|
|
raise PanelError(
|
|
503,
|
|
'Userbot common Secret or ConfigMap is missing in the userbot namespace',
|
|
) from exc
|
|
raise self._api_error(exc, 'Could not verify userbot prerequisites') from exc
|
|
|
|
def list_instances(
|
|
self,
|
|
query: str = '',
|
|
status: str = '',
|
|
) -> list[InstanceSummary]:
|
|
instances: list[InstanceSummary] = []
|
|
for namespace in (self.settings.namespace, *self.settings.legacy_namespaces):
|
|
try:
|
|
deployments = self.apps.list_namespaced_deployment(
|
|
namespace,
|
|
label_selector=MANAGED_LABEL,
|
|
).items
|
|
except ApiException as exc:
|
|
raise self._api_error(exc, f'Could not list Deployments in {namespace}') from exc
|
|
instances.extend(self._summarize(namespace, deployment) for deployment in deployments)
|
|
|
|
query = query.strip().lower()
|
|
if query:
|
|
instances = [
|
|
item
|
|
for item in instances
|
|
if query in item.instance_id.lower()
|
|
or query in item.display_name.lower()
|
|
or query in (item.pod or '').lower()
|
|
]
|
|
if status:
|
|
instances = [item for item in instances if item.status == status]
|
|
return sorted(instances, key=lambda item: (item.legacy, item.instance_id))
|
|
|
|
def get_instance(self, instance_id: str) -> InstanceSummary:
|
|
namespace, deployment = self._find_deployment(instance_id)
|
|
return self._summarize(namespace, deployment)
|
|
|
|
def assert_available(self, instance_id: str) -> None:
|
|
names = self._resource_names(instance_id)
|
|
checks = (
|
|
(self.apps.read_namespaced_deployment, names['deployment'], 'Deployment'),
|
|
(self.core.read_namespaced_secret, names['secret'], 'Secret'),
|
|
(self.core.read_namespaced_persistent_volume_claim, names['pvc'], 'PVC'),
|
|
)
|
|
for read, name, kind in checks:
|
|
try:
|
|
read(name, self.settings.namespace)
|
|
except ApiException as exc:
|
|
if exc.status == 404:
|
|
continue
|
|
raise self._api_error(exc, f'Could not check {kind} {name}') from exc
|
|
raise PanelError(409, f'{kind} {name} already exists')
|
|
|
|
def provision(self, account: AccountBase, session_string: str) -> InstanceSummary:
|
|
with self._provision_lock:
|
|
self.assert_available(account.instance_id)
|
|
names = self._resource_names(account.instance_id)
|
|
created: list[tuple[str, str]] = []
|
|
try:
|
|
self.core.create_namespaced_secret(
|
|
self.settings.namespace,
|
|
self._secret(account, session_string, names),
|
|
)
|
|
created.append(('secret', names['secret']))
|
|
self.core.create_namespaced_persistent_volume_claim(
|
|
self.settings.namespace,
|
|
self._pvc(account, names),
|
|
)
|
|
created.append(('pvc', names['pvc']))
|
|
self.apps.create_namespaced_deployment(
|
|
self.settings.namespace,
|
|
self._deployment(account, names),
|
|
)
|
|
created.append(('deployment', names['deployment']))
|
|
except ApiException as exc:
|
|
self._rollback(created)
|
|
if exc.status == 409:
|
|
raise PanelError(
|
|
409,
|
|
f'Instance {account.instance_id} already exists',
|
|
) from exc
|
|
raise self._api_error(exc, 'Could not create userbot instance') from exc
|
|
return self.get_instance(account.instance_id)
|
|
|
|
def scale(self, instance_id: str, replicas: int) -> InstanceSummary:
|
|
namespace, deployment = self._find_deployment(instance_id)
|
|
try:
|
|
self.apps.patch_namespaced_deployment_scale(
|
|
deployment.metadata.name,
|
|
namespace,
|
|
{'spec': {'replicas': replicas}},
|
|
)
|
|
except ApiException as exc:
|
|
raise self._api_error(exc, 'Could not scale userbot instance') from exc
|
|
return self.get_instance(instance_id)
|
|
|
|
def restart(self, instance_id: str) -> InstanceSummary:
|
|
namespace, deployment = self._find_deployment(instance_id)
|
|
if (deployment.spec.replicas or 0) == 0:
|
|
raise PanelError(409, 'Stopped instance cannot be restarted')
|
|
timestamp = datetime.now(UTC).isoformat()
|
|
try:
|
|
self.apps.patch_namespaced_deployment(
|
|
deployment.metadata.name,
|
|
namespace,
|
|
{
|
|
'spec': {
|
|
'template': {
|
|
'metadata': {
|
|
'annotations': {RESTART_ANNOTATION: timestamp},
|
|
}
|
|
}
|
|
}
|
|
},
|
|
)
|
|
except ApiException as exc:
|
|
raise self._api_error(exc, 'Could not restart userbot instance') from exc
|
|
return self.get_instance(instance_id)
|
|
|
|
def logs(self, instance_id: str, tail: int = 250) -> str:
|
|
namespace, deployment = self._find_deployment(instance_id)
|
|
pods = self._pods_for_deployment(namespace, deployment)
|
|
if not pods:
|
|
raise PanelError(409, 'Userbot Pod is not running')
|
|
pod = pods[0]
|
|
container = deployment.spec.template.spec.containers[0].name
|
|
try:
|
|
return self.core.read_namespaced_pod_log(
|
|
pod.metadata.name,
|
|
namespace,
|
|
container=container,
|
|
tail_lines=max(1, min(tail, 1000)),
|
|
timestamps=True,
|
|
)
|
|
except ApiException as exc:
|
|
raise self._api_error(exc, 'Could not read userbot logs') from exc
|
|
|
|
def delete(self, instance_id: str, *, delete_data: bool) -> None:
|
|
namespace, deployment = self._find_deployment(instance_id)
|
|
annotations = deployment.metadata.annotations or {}
|
|
if annotations.get(LEGACY_ANNOTATION) == 'true' or namespace != self.settings.namespace:
|
|
raise PanelError(409, 'Legacy instances cannot be deleted from the panel')
|
|
|
|
names = self._resource_names(instance_id)
|
|
secret_name = annotations.get(CREDENTIALS_ANNOTATION, names['secret'])
|
|
pvc_name = annotations.get(PVC_ANNOTATION, names['pvc'])
|
|
operations = [
|
|
(
|
|
self.apps.delete_namespaced_deployment,
|
|
(deployment.metadata.name, namespace),
|
|
{'propagation_policy': 'Foreground'},
|
|
),
|
|
(self.core.delete_namespaced_secret, (secret_name, namespace), {}),
|
|
]
|
|
if delete_data:
|
|
operations.append(
|
|
(
|
|
self.core.delete_namespaced_persistent_volume_claim,
|
|
(pvc_name, namespace),
|
|
{},
|
|
)
|
|
)
|
|
for delete_resource, args, kwargs in operations:
|
|
try:
|
|
delete_resource(*args, **kwargs)
|
|
except ApiException as exc:
|
|
if exc.status != 404:
|
|
raise self._api_error(exc, 'Could not delete userbot instance') from exc
|
|
|
|
def _find_deployment(self, instance_id: str) -> tuple[str, Any]:
|
|
selector = f'{MANAGED_LABEL},{INSTANCE_LABEL}={instance_id}'
|
|
for namespace in (self.settings.namespace, *self.settings.legacy_namespaces):
|
|
try:
|
|
items = self.apps.list_namespaced_deployment(
|
|
namespace,
|
|
label_selector=selector,
|
|
).items
|
|
except ApiException as exc:
|
|
raise self._api_error(exc, 'Could not find userbot instance') from exc
|
|
if items:
|
|
return namespace, items[0]
|
|
raise PanelError(404, f'Instance {instance_id} does not exist')
|
|
|
|
def _summarize(self, namespace: str, deployment: Any) -> InstanceSummary:
|
|
labels = deployment.metadata.labels or {}
|
|
annotations = deployment.metadata.annotations or {}
|
|
instance_id = labels.get(INSTANCE_LABEL, deployment.metadata.name)
|
|
legacy = annotations.get(LEGACY_ANNOTATION) == 'true'
|
|
pods = self._pods_for_deployment(namespace, deployment)
|
|
pod = pods[0] if pods else None
|
|
desired = deployment.spec.replicas or 0
|
|
ready = False
|
|
reason: str | None = None
|
|
restarts = 0
|
|
updated_at = deployment.metadata.creation_timestamp
|
|
|
|
if pod is not None:
|
|
statuses = pod.status.container_statuses or []
|
|
ready = bool(statuses) and all(item.ready for item in statuses)
|
|
restarts = sum(item.restart_count for item in statuses)
|
|
for item in statuses:
|
|
state = item.state
|
|
if state and state.waiting and state.waiting.reason:
|
|
reason = state.waiting.reason
|
|
break
|
|
if state and state.terminated and state.terminated.reason:
|
|
reason = state.terminated.reason
|
|
break
|
|
updated_at = pod.status.start_time or pod.metadata.creation_timestamp
|
|
|
|
if desired == 0:
|
|
status = 'stopped'
|
|
elif reason in {
|
|
'CrashLoopBackOff',
|
|
'Error',
|
|
'ImagePullBackOff',
|
|
'ErrImagePull',
|
|
'CreateContainerConfigError',
|
|
'RunContainerError',
|
|
} or (pod is not None and pod.status.phase == 'Failed'):
|
|
status = 'error'
|
|
elif ready and (deployment.status.available_replicas or 0) > 0:
|
|
status = 'running'
|
|
else:
|
|
status = 'pending'
|
|
reason = reason or (pod.status.phase if pod is not None else 'Scheduling')
|
|
|
|
container_spec = deployment.spec.template.spec.containers[0]
|
|
limits = (container_spec.resources.limits or {}) if container_spec.resources else {}
|
|
pvc_name = annotations.get(PVC_ANNOTATION) or self._deployment_pvc(deployment)
|
|
storage = self._pvc_storage(namespace, pvc_name)
|
|
cpu_usage, memory_usage = self._pod_metrics(namespace, pod.metadata.name if pod else None)
|
|
return InstanceSummary(
|
|
instance_id=instance_id,
|
|
display_name=annotations.get(DISPLAY_ANNOTATION, instance_id),
|
|
namespace=namespace,
|
|
deployment=deployment.metadata.name,
|
|
pod=pod.metadata.name if pod else None,
|
|
status=status,
|
|
reason=reason,
|
|
ready=ready,
|
|
restarts=restarts,
|
|
pvc=pvc_name,
|
|
storage=storage,
|
|
image=container_spec.image,
|
|
cpu_limit=limits.get('cpu'),
|
|
memory_limit=limits.get('memory'),
|
|
cpu_usage=cpu_usage,
|
|
memory_usage=memory_usage,
|
|
updated_at=_as_datetime(updated_at),
|
|
legacy=legacy,
|
|
deletable=not legacy and namespace == self.settings.namespace,
|
|
)
|
|
|
|
def _pods_for_deployment(self, namespace: str, deployment: Any) -> list[Any]:
|
|
selector = _selector(deployment.spec.selector.match_labels)
|
|
try:
|
|
pods = self.core.list_namespaced_pod(
|
|
namespace,
|
|
label_selector=selector,
|
|
).items
|
|
except ApiException as exc:
|
|
raise self._api_error(exc, 'Could not list userbot Pods') from exc
|
|
return sorted(
|
|
pods,
|
|
key=lambda pod: pod.metadata.creation_timestamp or datetime.min.replace(tzinfo=UTC),
|
|
reverse=True,
|
|
)
|
|
|
|
def _pod_metrics(self, namespace: str, pod_name: str | None) -> tuple[str | None, str | None]:
|
|
if not pod_name:
|
|
return None, None
|
|
try:
|
|
metrics = self.custom.get_namespaced_custom_object(
|
|
'metrics.k8s.io',
|
|
'v1beta1',
|
|
namespace,
|
|
'pods',
|
|
pod_name,
|
|
)
|
|
except (ApiException, AttributeError):
|
|
return None, None
|
|
containers = metrics.get('containers', [])
|
|
cpu = containers[0].get('usage', {}).get('cpu') if containers else None
|
|
memory = containers[0].get('usage', {}).get('memory') if containers else None
|
|
return cpu, memory
|
|
|
|
def _pvc_storage(self, namespace: str, pvc_name: str | None) -> str | None:
|
|
if not pvc_name:
|
|
return None
|
|
try:
|
|
pvc = self.core.read_namespaced_persistent_volume_claim(pvc_name, namespace)
|
|
except ApiException:
|
|
return None
|
|
requests = pvc.spec.resources.requests or {}
|
|
return requests.get('storage')
|
|
|
|
@staticmethod
|
|
def _deployment_pvc(deployment: Any) -> str | None:
|
|
for volume in deployment.spec.template.spec.volumes or []:
|
|
if volume.persistent_volume_claim:
|
|
return volume.persistent_volume_claim.claim_name
|
|
return None
|
|
|
|
@staticmethod
|
|
def _resource_names(instance_id: str) -> dict[str, str]:
|
|
return {
|
|
'deployment': f'userbot-{instance_id}',
|
|
'secret': f'userbot-{instance_id}-credentials',
|
|
'pvc': f'userbot-{instance_id}-data',
|
|
}
|
|
|
|
def _metadata(
|
|
self,
|
|
account: AccountBase,
|
|
names: dict[str, str],
|
|
resource_name: str,
|
|
) -> client.V1ObjectMeta:
|
|
return client.V1ObjectMeta(
|
|
name=resource_name,
|
|
namespace=self.settings.namespace,
|
|
labels={
|
|
'app.kubernetes.io/name': 'userbot',
|
|
INSTANCE_LABEL: account.instance_id,
|
|
MANAGED_BY_LABEL: 'userbot-panel',
|
|
},
|
|
annotations={
|
|
DISPLAY_ANNOTATION: account.display_name,
|
|
CREDENTIALS_ANNOTATION: names['secret'],
|
|
PVC_ANNOTATION: names['pvc'],
|
|
},
|
|
)
|
|
|
|
def _secret(
|
|
self,
|
|
account: AccountBase,
|
|
session_string: str,
|
|
names: dict[str, str],
|
|
) -> client.V1Secret:
|
|
return client.V1Secret(
|
|
metadata=self._metadata(account, names, names['secret']),
|
|
type='Opaque',
|
|
string_data={
|
|
'API_ID': str(account.api_id),
|
|
'API_HASH': account.api_hash,
|
|
'STRINGSESSION': session_string,
|
|
},
|
|
)
|
|
|
|
def _pvc(
|
|
self,
|
|
account: AccountBase,
|
|
names: dict[str, str],
|
|
) -> client.V1PersistentVolumeClaim:
|
|
return client.V1PersistentVolumeClaim(
|
|
metadata=self._metadata(account, names, names['pvc']),
|
|
spec=client.V1PersistentVolumeClaimSpec(
|
|
access_modes=['ReadWriteOnce'],
|
|
storage_class_name=self.settings.storage_class,
|
|
resources=client.V1VolumeResourceRequirements(
|
|
requests={'storage': account.resources.storage},
|
|
),
|
|
),
|
|
)
|
|
|
|
def _deployment(
|
|
self,
|
|
account: AccountBase,
|
|
names: dict[str, str],
|
|
) -> client.V1Deployment:
|
|
pod_labels = {
|
|
'app.kubernetes.io/name': 'userbot',
|
|
INSTANCE_LABEL: account.instance_id,
|
|
MANAGED_BY_LABEL: 'userbot-panel',
|
|
}
|
|
container = client.V1Container(
|
|
name='userbot',
|
|
image=self.settings.image,
|
|
image_pull_policy='Always',
|
|
env_from=[
|
|
client.V1EnvFromSource(secret_ref=client.V1SecretEnvSource(name=self.settings.common_secret)),
|
|
client.V1EnvFromSource(config_map_ref=client.V1ConfigMapEnvSource(name=self.settings.common_config)),
|
|
client.V1EnvFromSource(secret_ref=client.V1SecretEnvSource(name=names['secret'])),
|
|
],
|
|
resources=client.V1ResourceRequirements(
|
|
requests={'cpu': '80m', 'memory': '512Mi'},
|
|
limits={
|
|
'cpu': account.resources.cpu_limit,
|
|
'memory': account.resources.memory_limit,
|
|
},
|
|
),
|
|
volume_mounts=[
|
|
client.V1VolumeMount(name='data', mount_path='/app/data'),
|
|
client.V1VolumeMount(name='downloads', mount_path='/app/downloads'),
|
|
],
|
|
)
|
|
pod_spec = client.V1PodSpec(
|
|
service_account_name='userbot-runtime',
|
|
automount_service_account_token=False,
|
|
containers=[container],
|
|
termination_grace_period_seconds=30,
|
|
volumes=[
|
|
client.V1Volume(
|
|
name='data',
|
|
persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(claim_name=names['pvc']),
|
|
),
|
|
client.V1Volume(
|
|
name='downloads',
|
|
host_path=client.V1HostPathVolumeSource(
|
|
path=self.settings.downloads_host_path,
|
|
type='DirectoryOrCreate',
|
|
),
|
|
),
|
|
],
|
|
)
|
|
return client.V1Deployment(
|
|
metadata=self._metadata(account, names, names['deployment']),
|
|
spec=client.V1DeploymentSpec(
|
|
replicas=1,
|
|
strategy=client.V1DeploymentStrategy(type='Recreate'),
|
|
selector=client.V1LabelSelector(match_labels=pod_labels),
|
|
template=client.V1PodTemplateSpec(
|
|
metadata=client.V1ObjectMeta(labels=pod_labels),
|
|
spec=pod_spec,
|
|
),
|
|
),
|
|
)
|
|
|
|
def _rollback(self, created: list[tuple[str, str]]) -> None:
|
|
for kind, name in reversed(created):
|
|
try:
|
|
if kind == 'deployment':
|
|
self.apps.delete_namespaced_deployment(name, self.settings.namespace)
|
|
elif kind == 'pvc':
|
|
self.core.delete_namespaced_persistent_volume_claim(
|
|
name,
|
|
self.settings.namespace,
|
|
)
|
|
else:
|
|
self.core.delete_namespaced_secret(name, self.settings.namespace)
|
|
except ApiException as exc:
|
|
logger.warning(
|
|
'Rollback of %s %s in %s failed: %s',
|
|
kind,
|
|
name,
|
|
self.settings.namespace,
|
|
exc,
|
|
)
|
|
|
|
@staticmethod
|
|
def _api_error(exc: ApiException, detail: str) -> PanelError:
|
|
if exc.status == 403:
|
|
return PanelError(503, f'{detail}: Kubernetes RBAC denied the operation')
|
|
if exc.status == 409:
|
|
return PanelError(409, f'{detail}: resource conflict')
|
|
if exc.status == 404:
|
|
return PanelError(404, f'{detail}: resource not found')
|
|
return PanelError(503, detail)
|