fix(server): harden deployment, media, state, jobs

- Lock down /media/: deny state.json, DBs, sessions; allowlist extensions incl. archives/docs (M-1)
- parse_bool() fixes; HEAD 404; shutdown drains queue; range edge cases (M-3, M-4, M-7, M-8)
- int() coercion -> 400; no filesystem paths in errors; path-only access log (M-19, L-1)
- Security headers, QR TTL 60s, trusted-host allowlist, legacy add/remove via update() (L-4, L-5, L-6, L-8)
- Clean continuous channels on import and migration; restart-during-drain; tombstone managers (F-1, F-3, F-4)
- Durability: fsync + unique tmp + stale sweep + 0600/0700 perms (M-10, M-18)
- Jobs run on dedicated loop thread; set_scrape_media passthrough; media chunked; state throttled;
  exact media file reuse; honest scrape failure status (M-11, M-12, M-13, M-14)
- Health aggregates per-account; legacy GETs delegate post-migration (M-15, M-9)
- k8s: runAsNonRoot 1000 + resource limits, no readOnlyRootFilesystem (M-16)
- UI: dropped-invalid and credentials-reentry toasts; swagger XSS-safe (F-2, L-9, L-2)
- CI: non-blocking pip-audit job in both workflows (L-3)
- 50 tests passing; REVIEW.md updated (C-1/M-20 won't fix: local-only by design)
This commit is contained in:
2026-09-07 12:36:33 +02:00
parent a2468a2a2c
commit e93e68db7e
13 changed files with 1182 additions and 189 deletions
+346 -76
View File
@@ -91,6 +91,74 @@ AUTH_ATTEMPTS_MAX_ENTRIES = 10_000
_auth_attempts: Dict[Tuple[str, str], Dict[str, Any]] = {}
_auth_attempts_lock = threading.Lock()
# ── M-3: boolean coercion helper ─────────────────────────────────────────
_TRUE_VALUES = {"true", "1", "yes", "on"}
_FALSE_VALUES = {"false", "0", "no", "off"}
def parse_bool(value: Any, default: bool = False) -> bool:
"""Coerce *value* to bool safely. Returns *default* for unrecognised input.
Recognises True/False, ``"true"``/``"false"``, ``"1"``/``"0"``,
``"yes"``/``"no"``, ``"on"``/``"off"`` (case-insensitive). Strings
outside these sets return *default* instead of silently being truthy.
"""
if isinstance(value, bool):
return value
if value is None:
return default
s = str(value).strip().lower()
if s in _TRUE_VALUES:
return True
if s in _FALSE_VALUES:
return False
return default
# ── L-8: trusted-host check ──────────────────────────────────────────────
import ipaddress # noqa: E402
def _is_trusted_host(host: str) -> bool:
"""Return True if *host* (the ``Host`` header value) is a loopback /
private address that this local-only deployment should trust."""
hostname = host.split("@")[-1].split(":")[0] # strip auth / port
if not hostname:
return False
if hostname in {"localhost", "127.0.0.1", "::1"}:
return True
try:
addr = ipaddress.ip_address(hostname)
return addr.is_loopback or addr.is_private or addr.is_link_local
except ValueError:
return False
# ── M-1: sensitive file / media extension allowlists ──────────────────────
_SENSITIVE_FILE_SUFFIXES = frozenset({
".db", ".session", ".db-wal", ".db-shm", ".db-journal",
})
_SENSITIVE_FILE_NAMES = frozenset({"state.json"})
# How long a generated QR login token/image is considered valid before it is
# dropped from the auth snapshot (seconds). L-5.
QR_TTL_SECONDS = 60
_MEDIA_FILE_EXTENSIONS = frozenset({
# images
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg", ".ico",
# video
".mp4", ".webm", ".mov", ".m4v", ".avi",
# audio
".mp3", ".ogg", ".wav", ".m4a", ".flac",
# documents / generic downloaded binary
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
".txt", ".csv", ".json", ".xml", ".bin",
# archives / other Telegram document types
".zip", ".rar", ".7z", ".apk", ".epub", ".tar", ".gz", ".bz2", ".xz",
".odt", ".ods", ".odp",
})
def _sweep_auth_attempts(now: float) -> None:
"""Evict expired lockout/cooldown entries when the dict grows too large."""
@@ -589,6 +657,23 @@ class JobRunner:
len(still_running),
timeout,
)
# M-7: The worker exits once _shutdown_flag is set, leaving any yet-to-be
# processed queued jobs stranded as "queued". Drain the queue and mark
# each leftover as failed so their event streams / polling terminate.
drained = 0
while True:
try:
job = self.queue.get_nowait()
except queue.Empty:
break
self.queue.task_done()
with self.lock:
job.status = "failed"
job.error = "Server shutting down; job was cancelled"
job.finished_at = utc_now_iso()
drained += 1
if drained:
logger.info("Failed %d queued job(s) during shutdown.", drained)
def _run(self) -> None:
while not self._shutdown_flag:
@@ -653,6 +738,7 @@ class TelegramAuthManager:
"details": "",
"qr_url": None,
"qr_image": None,
"qr_created_at": None,
"phone": None,
"phone_code_hash": None,
"qr_login": None,
@@ -713,6 +799,16 @@ class TelegramAuthManager:
with self.lock:
data = self._get_auth_data(account_id)
snapshot = dict(data)
# L-5: expire a QR login token that has not been scanned within
# QR_TTL_SECONDS so it cannot live forever in the auth snapshot.
qr_created = data.get("qr_created_at")
if qr_created and (time.time() - float(qr_created)) > QR_TTL_SECONDS:
if data.get("qr_url") or data.get("qr_image"):
data["qr_url"] = None
data["qr_image"] = None
data["qr_created_at"] = None
snapshot = dict(data)
snapshot["qr_expired"] = True
snapshot.pop("qr_login", None)
snapshot.pop("qr_wait_task", None)
snapshot.pop("phone_code_hash", None)
@@ -768,6 +864,7 @@ class TelegramAuthManager:
details="Scan the QR code in Telegram: Settings -> Devices -> Scan QR.",
qr_url=qr_url,
qr_image=self._make_qr_image(qr_url),
qr_created_at=time.time(),
)
with self.lock:
data = self._get_auth_data(account_id)
@@ -792,6 +889,7 @@ class TelegramAuthManager:
details="Telegram session is authorized.",
qr_url=None,
qr_image=None,
qr_created_at=None,
phone=None,
)
except SessionPasswordNeededError:
@@ -809,6 +907,7 @@ class TelegramAuthManager:
details=f"QR login failed: {exc}",
qr_url=None,
qr_image=None,
qr_created_at=None,
)
def start_qr_login(self, account_id: str) -> Dict[str, Any]:
@@ -859,6 +958,7 @@ class TelegramAuthManager:
details="Telegram session is authorized.",
qr_url=None,
qr_image=None,
qr_created_at=None,
)
except SessionPasswordNeededError:
self._set_state(
@@ -882,6 +982,7 @@ class TelegramAuthManager:
details="Telegram session is authorized.",
qr_url=None,
qr_image=None,
qr_created_at=None,
)
return self.auth_state(account_id)
@@ -937,6 +1038,10 @@ class PerAccountContinuousScrapeManager:
self.lock = threading.RLock()
self.thread: Optional[threading.Thread] = None
self.stop_event = threading.Event()
# F-4: set on remove_account when its thread could not be joined in
# time; the manager is kept as a tombstone so it is reused (after the
# drain) instead of spawning a duplicate worker on account re-add.
self._removing = False
self.config: Dict[str, Any] = self._load_config()
self.status: Dict[str, Any] = {
"running": False,
@@ -1053,9 +1158,34 @@ class PerAccountContinuousScrapeManager:
def start(self) -> None:
with self.lock:
if self.status["running"]:
self._log("Continuous scraping is already running.", "warn")
thread = self.thread
if thread is not None and thread.is_alive():
if self.stop_event.is_set():
# F-3: a stop was requested and the old thread is still
# draining (its finally has not yet flipped running=False).
# Wait for it to exit before starting a fresh one so the
# account does not end up enabled=True with a dead thread.
self._log(
"Previous scrape thread is stopping; waiting before restart.",
"warn",
)
else:
# A live thread is genuinely running — do NOT spawn a
# duplicate.
self._log("Continuous scraping is already running.", "warn")
return
if thread is not None and thread.is_alive():
# Join outside the lock (bounded) so the draining thread can mark
# itself finished. If it does not exit in time, refuse to start a
# duplicate worker rather than risk overlapping writes.
thread.join(timeout=5.0)
if thread.is_alive():
self._log(
"Previous scrape thread still stopping; cannot restart yet.",
"error",
)
return
with self.lock:
self.stop_event.clear()
self.status["running"] = True
self.status["last_started_at"] = utc_now_iso()
@@ -1180,9 +1310,18 @@ class ContinuousScrapeOrchestrator:
def _get_or_create(self, account_id: str) -> PerAccountContinuousScrapeManager:
with self.lock:
if account_id not in self.managers:
self.managers[account_id] = PerAccountContinuousScrapeManager(account_id)
return self.managers[account_id]
mgr = self.managers.get(account_id)
if mgr is None:
mgr = PerAccountContinuousScrapeManager(account_id)
self.managers[account_id] = mgr
elif mgr._removing:
# F-4: the manager was left as a tombstone because its worker
# could not be joined during remove_account. The account is
# being re-added / accessed, so clear the tombstone and reuse
# the manager (start() waits for the old thread to drain before
# spawning a fresh one, avoiding duplicate workers).
mgr._removing = False
return mgr
def start_account(self, account_id: str) -> None:
mgr = self._get_or_create(account_id)
@@ -1240,18 +1379,24 @@ class ContinuousScrapeOrchestrator:
mgr.stop()
# Wait for the scrape thread to actually stop before the caller
# deletes the account directory / session files, so rmtree does not
# race with a writer mid-iteration. If the thread is still running
# (e.g. mid-scrape) after the timeout we proceed best-effort and
# log a warning.
if not mgr.join(timeout=REMOVE_ACCOUNT_JOIN_TIMEOUT):
# race with a writer mid-iteration.
if mgr.join(timeout=REMOVE_ACCOUNT_JOIN_TIMEOUT):
# Thread finished: safe to drop the manager.
with self.lock:
self.managers.pop(account_id, None)
else:
# F-4: the thread is still draining. Do NOT pop the manager —
# leave it as a tombstone (marked removing) so a later re-add of
# the same account id reuses it / waits for the drain instead of
# spawning a duplicate worker that writes the same dirs.
logger.warning(
"Continuous scrape thread for account %r still running after "
"%.1fs; removing account data best-effort",
"%.1fs; keeping manager as tombstone",
account_id,
REMOVE_ACCOUNT_JOIN_TIMEOUT,
)
with self.lock:
self.managers.pop(account_id, None)
with self.lock:
mgr._removing = True
def stop_all(self) -> None:
with self.lock:
@@ -1336,18 +1481,31 @@ def auth_status() -> Dict[str, Any]:
return auth_status_for(account_id=None)
def dashboard_payload(job_runner: JobRunner) -> Dict[str, Any]:
state = load_state()
channels = list_channels_snapshot()
def dashboard_payload(job_runner: JobRunner, account_id: Optional[str] = None) -> Dict[str, Any]:
if account_id:
acc_state = load_account(DATA_DIR, account_id)
state = {
"scrape_media": bool(acc_state.get("scrape_media", True)),
"channel_count": len(acc_state.get("channels", {})),
"forwarding_rules": acc_state.get("forwarding_rules", []),
}
channels = list_channels_snapshot(account_id)
auth = auth_status_for(account_id)
jobs = job_runner.recent_jobs(account_id=account_id)
else:
state = load_state()
channels = list_channels_snapshot()
auth = auth_status()
jobs = job_runner.recent_jobs()
return {
"state": {
"scrape_media": bool(state.get("scrape_media", True)),
"channel_count": len(state.get("channels", {})),
"forwarding_rules": state.get("forwarding_rules", []),
},
"auth": auth_status(),
"auth": auth,
"channels": channels,
"jobs": job_runner.recent_jobs(),
"jobs": jobs,
}
@@ -2050,7 +2208,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/openapi.json":
return self.send_json(openapi_payload())
if path == "/api/dashboard":
return self.send_json(dashboard_payload(self.app.job_runner))
return self.send_json(
dashboard_payload(self.app.job_runner, self.app.legacy_account_id)
)
if path == "/api/auth":
if self.app.legacy_account_id:
return self.send_json(self.app.auth_manager.auth_state(self.app.legacy_account_id))
@@ -2094,27 +2254,34 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found")
return self.send_json(job)
if path == "/api/channels":
return self.send_json(list_channels_snapshot())
return self.send_json(list_channels_snapshot(self.app.legacy_account_id))
if path.startswith("/api/channels/") and path.endswith("/messages"):
parts = path.split("/")
try:
channel_id = normalize_channel_id(urllib.parse.unquote(parts[3]))
except ValueError as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
try:
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
except (TypeError, ValueError):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'limit' query parameter")
before = query.get("before")
before_message_id = int(before[0]) if before else None
try:
before_message_id = int(before[0]) if before else None
except (TypeError, ValueError):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'before' query parameter")
search = (query.get("search") or query.get("q") or [""])[0].strip()
account_id = self.app.legacy_account_id
payload = {
"channel_id": channel_id,
"messages": load_messages(
None, channel_id, limit=limit, before_message_id=before_message_id,
account_id, channel_id, limit=limit, before_message_id=before_message_id,
search=search or None
),
"channel": next(
(
item
for item in list_channels_snapshot()
for item in list_channels_snapshot(account_id)
if item["channel_id"] == channel_id
),
None,
@@ -2258,9 +2425,15 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
def _handle_get_account_channel_messages(
self, account_id: str, channel_id: str, query: Dict[str, List[str]]
) -> None:
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
try:
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
except (TypeError, ValueError):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'limit' query parameter")
before = query.get("before")
before_message_id = int(before[0]) if before else None
try:
before_message_id = int(before[0]) if before else None
except (TypeError, ValueError):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'before' query parameter")
search = (query.get("search") or query.get("q") or [""])[0].strip()
payload = {
"channel_id": channel_id,
@@ -2316,6 +2489,12 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
or (path.startswith("/api/channels/") and path.endswith("/messages"))
or path.startswith("/api/accounts/")
):
# M-4: for the job-events route, mirror GET's behavior and verify
# the job actually exists before responding with 200.
if path.startswith("/api/jobs/") and path.endswith("/events"):
job_id = path.split("/")[-2] if path.endswith("/events") else None
if job_id and not self.app.job_runner.get_job(job_id):
return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found")
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.end_headers()
@@ -2339,22 +2518,23 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
# ── Legacy endpoints ────────────────────────────────────────────
if path == "/api/settings/media":
value = bool(body.get("value"))
value = parse_bool(body.get("value"))
if self.app.legacy_account_id:
return self._handle_account_settings_media(self.app.legacy_account_id, value)
job = self.app.job_runner.create_job(
"set_scrape_media",
"Update media scraping setting",
{"value": value},
)
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
# No legacy account: persist to the (global) legacy store directly.
# The ScraperJobService set_scrape_media job is a passthrough, so
# enqueueing it alone would silently drop the setting.
def _media_mutate(global_state: Dict[str, Any]) -> None:
global_state["scrape_media"] = value
STATE_STORE.update(_media_mutate)
return self.send_json({"ok": True, "scrape_media": value})
if path == "/api/continuous":
try:
enabled = bool(body.get("enabled"))
enabled = parse_bool(body.get("enabled"))
interval_minutes = int(body.get("interval_minutes", 1))
channels, dropped = clean_continuous_channels(body.get("channels", []))
run_all_tracked = bool(body.get("run_all_tracked", True))
run_all_tracked = parse_bool(body.get("run_all_tracked", True), default=True)
if self.app.legacy_account_id:
payload = self.app.continuous_orchestrator.update_for(
account_id=self.app.legacy_account_id,
@@ -2366,7 +2546,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
else:
payload = {"config": {}, "status": {"running": False, "logs": [], "log_entries": []}}
except Exception as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(
exc, fallback="Failed to update continuous scraping"
)
if isinstance(payload, dict):
payload["dropped_invalid"] = dropped
return self.send_json(payload)
@@ -2383,7 +2565,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
save_state(state)
payload = self._legacy_auth_snapshot()
except Exception as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(
exc, fallback="Failed to save API credentials"
)
return self.send_json(payload)
if path == "/api/auth/qr/start":
@@ -2405,12 +2589,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
if self.app.legacy_account_id:
return self._handle_account_channel_add(self.app.legacy_account_id, channel_id, body.get("name"))
state = load_state()
if channel_id not in state["channels"]:
state["channels"][channel_id] = 0
if body.get("name"):
state.setdefault("channel_names", {})[channel_id] = str(body["name"]).strip()
save_state(state)
# L-6: use the atomic mutator to avoid lost-update races.
def _add_mutate(state: Dict[str, Any]) -> None:
if channel_id not in state.get("channels", {}):
state.setdefault("channels", {})[channel_id] = 0
if body.get("name"):
state.setdefault("channel_names", {})[channel_id] = str(body["name"]).strip()
STATE_STORE.update(_add_mutate)
return self.send_json({"ok": True, "channel_id": channel_id})
if path == "/api/channels/remove":
@@ -2420,11 +2605,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
if self.app.legacy_account_id:
return self._handle_account_channel_remove(self.app.legacy_account_id, channel_id)
state = load_state()
existed = channel_id in state.get("channels", {})
state.get("channels", {}).pop(channel_id, None)
save_state(state)
return self.send_json({"ok": existed, "channel_id": channel_id})
# L-6: use the atomic mutator to avoid lost-update races.
existed = [False]
def _remove_mutate(state: Dict[str, Any]) -> None:
chans = state.get("channels", {})
if channel_id in chans:
existed[0] = True
del chans[channel_id]
state.get("channel_names", {}).pop(channel_id, None)
STATE_STORE.update(_remove_mutate)
return self.send_json({"ok": existed[0], "channel_id": channel_id})
if path == "/api/jobs/scrape":
channel_id = body.get("channel_id")
@@ -2581,19 +2771,30 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if not isinstance(state, dict):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "state object is required")
# F-1: normalise the imported continuous channels and drop invalid
# entries (path traversal / control chars) before persisting.
cs = state.get("continuous_scraping")
cs = cs if isinstance(cs, dict) else {}
cs_channels, _dropped = clean_continuous_channels(cs.get("channels", []))
try:
interval_minutes = int(cs.get("interval_minutes", 1) or 1)
except (TypeError, ValueError):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "interval_minutes must be a valid integer")
imported_state = {
"label": str(body.get("label") or state.get("label") or account_id).strip(),
"api_id": state.get("api_id"),
"api_hash": state.get("api_hash"),
"channels": self._clean_imported_channels(state.get("channels")),
"channel_names": self._clean_imported_channel_names(state.get("channel_names")),
"scrape_media": bool(state.get("scrape_media", True)),
"scrape_media": parse_bool(state.get("scrape_media", True), default=True),
"forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [],
"continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else {
"enabled": False,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
"continuous_scraping": {
"enabled": parse_bool(cs.get("enabled", False)),
"interval_minutes": interval_minutes,
"channels": cs_channels,
"run_all_tracked": parse_bool(cs.get("run_all_tracked", True), default=True),
},
}
@@ -2645,7 +2846,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if sub == ["jobs", "refresh-dialogs"]:
return self._handle_account_job_refresh_dialogs(account_id)
if sub == ["settings", "media"]:
return self._handle_account_settings_media(account_id, bool(body.get("value")))
return self._handle_account_settings_media(account_id, parse_bool(body.get("value")))
if sub == ["continuous"]:
return self._handle_account_continuous(account_id, body)
return self.send_error_json(HTTPStatus.NOT_FOUND, "Not found")
@@ -2664,14 +2865,14 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
try:
payload = self.app.auth_manager.save_credentials(account_id, int(api_id), api_hash)
except Exception as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(exc, fallback="Failed to save API credentials")
return self.send_json(payload)
def _handle_post_account_auth_qr_start(self, account_id: str) -> None:
try:
payload = self.app.auth_manager.start_qr_login(account_id)
except Exception as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(exc, fallback="Failed to start QR login")
return self.send_json(payload)
def _handle_post_account_auth_phone_request(self, account_id: str, body: Dict[str, Any]) -> None:
@@ -2693,7 +2894,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
payload = self.app.auth_manager.request_phone_code(account_id, phone)
except Exception as exc:
_record_auth_failure(ip, account_id)
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(exc, fallback="Failed to request phone code")
_record_auth_code_request(ip, account_id)
return self.send_json(payload)
@@ -2711,7 +2912,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
payload = self.app.auth_manager.submit_phone_code(account_id, code)
except Exception as exc:
_record_auth_failure(ip, account_id)
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(exc, fallback="Failed to submit phone code")
_record_auth_success(ip, account_id)
return self.send_json(payload)
@@ -2729,7 +2930,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
payload = self.app.auth_manager.submit_password(account_id, password)
except Exception as exc:
_record_auth_failure(ip, account_id)
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(exc, fallback="Failed to submit password")
_record_auth_success(ip, account_id)
return self.send_json(payload)
@@ -2875,10 +3076,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
def _handle_account_continuous(self, account_id: str, body: Dict[str, Any]) -> None:
try:
enabled = bool(body.get("enabled"))
enabled = parse_bool(body.get("enabled"))
interval_minutes = int(body.get("interval_minutes", 1))
channels, dropped = clean_continuous_channels(body.get("channels", []))
run_all_tracked = bool(body.get("run_all_tracked", True))
run_all_tracked = parse_bool(body.get("run_all_tracked", True), default=True)
payload = self.app.continuous_orchestrator.update_for(
account_id=account_id,
enabled=enabled,
@@ -2887,7 +3088,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
run_all_tracked=run_all_tracked,
)
except Exception as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
return self.send_bad_request_from_exc(exc, fallback="Failed to update continuous scraping")
payload["dropped_invalid"] = dropped
return self.send_json(payload)
@@ -2983,6 +3184,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if not origin_host or (host and origin_host != host):
self.send_error_json(HTTPStatus.FORBIDDEN, "Cross-origin request rejected")
return False
# L-8: When an Origin is present, guard against DNS rebinding: the
# Host must resolve to a local/private address, otherwise a remote
# attacker domain could make both Host == Origin pass. Requests
# without an Origin (curl, plain LAN tools) are unaffected.
if host and not _is_trusted_host(host):
self.send_error_json(HTTPStatus.FORBIDDEN, "Untrusted host")
return False
sec_fetch_site = self.headers.get("Sec-Fetch-Site", "").strip().lower()
if sec_fetch_site and sec_fetch_site not in {"same-origin", "same-site", "none"}:
self.send_error_json(HTTPStatus.FORBIDDEN, "Cross-origin request rejected")
@@ -3007,6 +3215,22 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
file_path.relative_to(DATA_DIR.resolve())
except ValueError:
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
# M-1: Never serve sensitive state / session / db files. Normal
# media documents live under DATA_DIR with a media-file extension;
# anything else (state.json, *.db, *.session, sqlite sidecars) is
# denied regardless of its location under DATA_DIR.
file_suffix = file_path.suffix.lower()
file_name = file_path.name.lower()
if file_name in _SENSITIVE_FILE_NAMES or file_suffix in _SENSITIVE_FILE_SUFFIXES:
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
# Only allow registered media/document extensions. This keeps the
# viewer's image/video/audio/document URLs working while blocking
# arbitrary file reads (e.g. /media/../state.json is already blocked
# by containment, and any other extension is not a media asset).
if file_suffix not in _MEDIA_FILE_EXTENSIONS:
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
if not file_path.exists() or not file_path.is_file():
return self.send_error_json(HTTPStatus.NOT_FOUND, "Media file not found")
mime_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
@@ -3018,6 +3242,11 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
last_modified = stat.st_mtime
range_header = self.headers.get("Range", "").strip()
# M-8: for an empty file there is nothing to range-serve; ignore the
# Range header entirely and send the full (empty) 200 response so
# clients don't see a spurious 416 for `bytes=0-0` on size-0 files.
if file_size == 0:
range_header = ""
if range_header.startswith("bytes="):
try:
start, end = self._parse_range(range_header, file_size)
@@ -3034,6 +3263,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
self.send_header("Content-Length", str(content_length))
self._send_security_headers()
if "text/html" in content_type:
self.send_header("Content-Security-Policy", self._CSP)
self.end_headers()
if not head_only:
self._write_file_range(file_path, start, content_length)
@@ -3046,24 +3278,42 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
)
self.send_header("Content-Length", str(file_size))
self._send_security_headers()
if "text/html" in content_type:
self.send_header("Content-Security-Policy", self._CSP)
self.end_headers()
if not head_only:
self._write_file_range(file_path, 0, file_size)
def _parse_range(self, range_header: str, file_size: int) -> tuple[int, int]:
range_val = range_header.removeprefix("bytes=").strip()
if "-" not in range_val:
raise ValueError("missing dash")
parts = range_val.split("-", 1)
if parts[0] == "":
end = int(parts[1])
start = max(0, file_size - end)
else:
start = int(parts[0])
end = int(parts[1]) if parts[1] else file_size - 1
if start < 0 or start >= file_size or end >= file_size or start > end:
raise ValueError("out of bounds")
return start, end
"""Parse a single ``bytes=start-end`` range spec.
Raises ``ValueError`` (cleanly caught by the caller → 416) for any
malformed / multi-range / out-of-bounds input. Never lets an
exception other than ValueError escape.
"""
try:
range_val = range_header.removeprefix("bytes=").strip()
# Multi-range requests ("bytes=0-1,5-6") are not supported here.
if "," in range_val:
raise ValueError("multi-range not supported")
if "-" not in range_val:
raise ValueError("missing dash")
parts = range_val.split("-", 1)
if parts[0] == "":
end = int(parts[1])
start = max(0, file_size - end)
else:
start = int(parts[0])
end = int(parts[1]) if parts[1] else file_size - 1
if start < 0 or start >= file_size or end >= file_size or start > end:
raise ValueError("out of bounds")
return start, end
except ValueError:
raise
except Exception as exc:
logger.debug("Unexpected range parse error: %s", exc)
raise ValueError("invalid range") from exc
def _write_file_range(self, file_path: Path, offset: int, length: int) -> None:
chunk_size = 65536
@@ -3077,22 +3327,42 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.wfile.write(chunk)
remaining -= len(chunk)
# ── L-4: Security headers ─────────────────────────────────────────────
_CSP = "default-src 'self'; img-src 'self' data:; style-src 'self'"
def _send_security_headers(self) -> None:
"""Emit standard hardening headers on every HTTP response."""
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("X-Frame-Options", "DENY")
self.send_header("Referrer-Policy", "no-referrer")
def send_json(self, payload: Any, status: HTTPStatus = HTTPStatus.OK) -> None:
raw = json.dumps(payload, ensure_ascii=False).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(raw)))
self._send_security_headers()
self.end_headers()
self.wfile.write(raw)
def send_error_json(self, status: HTTPStatus, message: str) -> None:
self.send_json({"error": message, "status": int(status)}, status=status)
def send_bad_request_from_exc(
self, exc: Exception, fallback: str = "Bad request"
) -> None:
"""Send a 400 JSON error without leaking filesystem paths or other
server internals. The caught exception is logged server-side and the
client receives only *fallback*."""
logger.exception("Request failed; client sees '%s'", fallback)
return self.send_error_json(HTTPStatus.BAD_REQUEST, fallback)
def log_message(self, format: str, *args: Any) -> None:
safe_path = self.path.split("?", 1)[0]
logger.info(
"%s %s%s",
self.command,
self.path,
safe_path,
args[1] if len(args) > 1 else "-",
)