fix(server): harden deployment, media, state, jobs
- Lock down /media/: deny state.json, DBs, sessions; allowlist extensions incl. archives/docs (M-1) - parse_bool() fixes; HEAD 404; shutdown drains queue; range edge cases (M-3, M-4, M-7, M-8) - int() coercion -> 400; no filesystem paths in errors; path-only access log (M-19, L-1) - Security headers, QR TTL 60s, trusted-host allowlist, legacy add/remove via update() (L-4, L-5, L-6, L-8) - Clean continuous channels on import and migration; restart-during-drain; tombstone managers (F-1, F-3, F-4) - Durability: fsync + unique tmp + stale sweep + 0600/0700 perms (M-10, M-18) - Jobs run on dedicated loop thread; set_scrape_media passthrough; media chunked; state throttled; exact media file reuse; honest scrape failure status (M-11, M-12, M-13, M-14) - Health aggregates per-account; legacy GETs delegate post-migration (M-15, M-9) - k8s: runAsNonRoot 1000 + resource limits, no readOnlyRootFilesystem (M-16) - UI: dropped-invalid and credentials-reentry toasts; swagger XSS-safe (F-2, L-9, L-2) - CI: non-blocking pip-audit job in both workflows (L-3) - 50 tests passing; REVIEW.md updated (C-1/M-20 won't fix: local-only by design)
This commit is contained in:
+134
-18
@@ -1,6 +1,8 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from copy import deepcopy
|
||||
@@ -71,15 +73,74 @@ class StateStore:
|
||||
|
||||
def save(self, state: Dict[str, Any]) -> None:
|
||||
with self.lock:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
merged = self._merge_defaults(state)
|
||||
with tmp_path.open("w", encoding="utf-8") as handle:
|
||||
json.dump(merged, handle, ensure_ascii=False, indent=2)
|
||||
handle.write("\n")
|
||||
tmp_path.replace(self.path)
|
||||
|
||||
# Unique temp file so two writers to the same path cannot clobber
|
||||
# each other's in-progress file, plus fsync before atomic rename so
|
||||
# a power loss cannot leave an empty/corrupt state file behind.
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=str(self.path.parent),
|
||||
prefix=self.path.name + ".tmp-",
|
||||
suffix=".tmp",
|
||||
)
|
||||
tmp_path = Path(tmp_name)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(merged, handle, ensure_ascii=False, indent=2)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
# Restrict permissions on the state file to the owning user.
|
||||
try:
|
||||
os.chmod(tmp_path, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp_path, self.path)
|
||||
finally:
|
||||
# Ensure no leftover stale temp file if something went wrong.
|
||||
if tmp_path.exists():
|
||||
try:
|
||||
tmp_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Best-effort directory fsync for full durability (POSIX only).
|
||||
try:
|
||||
dir_fd = os.open(str(self.path.parent), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(dir_fd)
|
||||
finally:
|
||||
os.close(dir_fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# After a successful save, best-effort sweep leftover stale tmp
|
||||
# files (e.g. from a crashed writer) older than an hour. Throttled
|
||||
# to avoid scanning the dir on every save.
|
||||
self._sweep_stale_tmp(now=time.time())
|
||||
|
||||
self._cache = None
|
||||
|
||||
_STALE_TMP_MAX_AGE = 3600.0 # 1 hour
|
||||
_STALE_TMP_SWEEP_INTERVAL = 60.0
|
||||
_stale_sweep_last: float = 0.0
|
||||
|
||||
def _sweep_stale_tmp(self, now: float) -> None:
|
||||
if (now - self._stale_sweep_last) < self._STALE_TMP_SWEEP_INTERVAL:
|
||||
return
|
||||
self._stale_sweep_last = now
|
||||
try:
|
||||
cutoff = now - self._STALE_TMP_MAX_AGE
|
||||
for stale in self.path.parent.glob(self.path.name + ".tmp-*.tmp"):
|
||||
try:
|
||||
if stale.stat().st_mtime < cutoff:
|
||||
stale.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def update(self, mutator: Callable[[Dict[str, Any]], None]) -> Dict[str, Any]:
|
||||
with self.lock:
|
||||
state = self.load()
|
||||
@@ -124,6 +185,18 @@ _GLOBAL_STORE: Optional[StateStore] = None
|
||||
|
||||
|
||||
def get_global_store(data_dir: Path) -> StateStore:
|
||||
"""Return the process-wide global state store singleton.
|
||||
|
||||
The store is cached in a module-global and the SAME instance is returned
|
||||
for the same process, regardless of how many times this is called with the
|
||||
same (or any) data_dir. webui_server's ``STATE_STORE`` should delegate to
|
||||
this function so there is exactly one authoritative global store per
|
||||
process rather than a second, potentially divergent instance.
|
||||
|
||||
Backward-compat note: callers that cache their own ``_GLOBAL_STORE=None``
|
||||
sentinel (e.g. tests resetting state between cases) still work because we
|
||||
re-create the singleton lazily on first call.
|
||||
"""
|
||||
global _GLOBAL_STORE
|
||||
if _GLOBAL_STORE is None:
|
||||
_GLOBAL_STORE = StateStore(data_dir / "state.json", defaults=GLOBAL_DEFAULTS)
|
||||
@@ -186,13 +259,47 @@ def account_session_path(session_dir: Path, account_id: str) -> str:
|
||||
# ── MIGRATION (with data copy) ─────────────────────────────────────────
|
||||
|
||||
|
||||
def _is_valid_channel_id(channel_id: str) -> bool:
|
||||
"""Path-safe channel id check.
|
||||
|
||||
Mirrors webui_server.normalize_channel_id's validation: reject entries
|
||||
containing ``/`` or ``\\``, control chars, ``.``/``..``, empty; keep
|
||||
numbers and plain names. Defined locally (not imported from webui_server,
|
||||
which would be circular) so it can be shared by migration.
|
||||
"""
|
||||
channel_id = str(channel_id or "").strip()
|
||||
if (
|
||||
not channel_id
|
||||
or "/" in channel_id
|
||||
or "\\" in channel_id
|
||||
or channel_id in {".", ".."}
|
||||
or any(ord(ch) < 32 for ch in channel_id)
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _clean_continuous_channels(channels: Any) -> List[str]:
|
||||
"""Normalize/drop invalid continuous-scraping channel entries during
|
||||
migration. Mirrors webui_server.clean_continuous_channels: strips a
|
||||
leading ``@``, keeps numbers/names, and drops unsafe entries so they can
|
||||
never become a path-traversal vector.
|
||||
"""
|
||||
cleaned: List[str] = []
|
||||
if not isinstance(channels, list):
|
||||
return cleaned
|
||||
for item in channels:
|
||||
cleaned.append(str(item).strip().lstrip("@"))
|
||||
return [c for c in cleaned if _is_valid_channel_id(c)]
|
||||
|
||||
|
||||
def _copy_channel_data(src_root: Path, dst_root: Path, channel_id: str) -> None:
|
||||
"""Copy a single channel's DB + media from src_root to dst_root."""
|
||||
src_ch = src_root / channel_id
|
||||
dst_ch = dst_root / channel_id
|
||||
if not src_ch.exists():
|
||||
return
|
||||
dst_ch.mkdir(parents=True, exist_ok=True)
|
||||
dst_ch.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
|
||||
# SQLite DB
|
||||
db_name = f"{channel_id}.db"
|
||||
@@ -206,7 +313,7 @@ def _copy_channel_data(src_root: Path, dst_root: Path, channel_id: str) -> None:
|
||||
src_media = src_ch / "media"
|
||||
dst_media = dst_ch / "media"
|
||||
if src_media.exists() and src_media.is_dir():
|
||||
dst_media.mkdir(parents=True, exist_ok=True)
|
||||
dst_media.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
for item in src_media.iterdir():
|
||||
if item.is_file():
|
||||
dst_file = dst_media / item.name
|
||||
@@ -250,7 +357,24 @@ def migrate_legacy_state(data_dir: Path, session_dir: Path) -> bool:
|
||||
|
||||
# ── 1. Create per-account state for "default" ──────────────────────
|
||||
acc_dir = data_dir / "accounts" / "default"
|
||||
acc_dir.mkdir(parents=True, exist_ok=True)
|
||||
acc_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
|
||||
# Normalize/drop invalid continuous-scraping channel entries during
|
||||
# migration so unsafe values can never become a path-traversal vector.
|
||||
continuous_cfg = raw.get(
|
||||
"continuous_scraping",
|
||||
{
|
||||
"enabled": True,
|
||||
"interval_minutes": 1,
|
||||
"channels": [],
|
||||
"run_all_tracked": True,
|
||||
},
|
||||
)
|
||||
if not isinstance(continuous_cfg, dict):
|
||||
continuous_cfg = {}
|
||||
continuous_cfg["channels"] = _clean_continuous_channels(
|
||||
continuous_cfg.get("channels")
|
||||
)
|
||||
|
||||
acc_state = {
|
||||
"label": "Default",
|
||||
@@ -260,15 +384,7 @@ def migrate_legacy_state(data_dir: Path, session_dir: Path) -> bool:
|
||||
"channel_names": raw.get("channel_names", {}),
|
||||
"scrape_media": raw.get("scrape_media", True),
|
||||
"forwarding_rules": raw.get("forwarding_rules", []),
|
||||
"continuous_scraping": raw.get(
|
||||
"continuous_scraping",
|
||||
{
|
||||
"enabled": True,
|
||||
"interval_minutes": 1,
|
||||
"channels": [],
|
||||
"run_all_tracked": True,
|
||||
},
|
||||
),
|
||||
"continuous_scraping": continuous_cfg,
|
||||
}
|
||||
|
||||
acc_state_path = acc_dir / "state.json"
|
||||
|
||||
Reference in New Issue
Block a user