fix(server): harden deployment, media, state, jobs
- Lock down /media/: deny state.json, DBs, sessions; allowlist extensions incl. archives/docs (M-1) - parse_bool() fixes; HEAD 404; shutdown drains queue; range edge cases (M-3, M-4, M-7, M-8) - int() coercion -> 400; no filesystem paths in errors; path-only access log (M-19, L-1) - Security headers, QR TTL 60s, trusted-host allowlist, legacy add/remove via update() (L-4, L-5, L-6, L-8) - Clean continuous channels on import and migration; restart-during-drain; tombstone managers (F-1, F-3, F-4) - Durability: fsync + unique tmp + stale sweep + 0600/0700 perms (M-10, M-18) - Jobs run on dedicated loop thread; set_scrape_media passthrough; media chunked; state throttled; exact media file reuse; honest scrape failure status (M-11, M-12, M-13, M-14) - Health aggregates per-account; legacy GETs delegate post-migration (M-15, M-9) - k8s: runAsNonRoot 1000 + resource limits, no readOnlyRootFilesystem (M-16) - UI: dropped-invalid and credentials-reentry toasts; swagger XSS-safe (F-2, L-9, L-2) - CI: non-blocking pip-audit job in both workflows (L-3) - 50 tests passing; REVIEW.md updated (C-1/M-20 won't fix: local-only by design)
This commit is contained in:
+346
-76
@@ -91,6 +91,74 @@ AUTH_ATTEMPTS_MAX_ENTRIES = 10_000
|
||||
_auth_attempts: Dict[Tuple[str, str], Dict[str, Any]] = {}
|
||||
_auth_attempts_lock = threading.Lock()
|
||||
|
||||
# ── M-3: boolean coercion helper ─────────────────────────────────────────
|
||||
_TRUE_VALUES = {"true", "1", "yes", "on"}
|
||||
_FALSE_VALUES = {"false", "0", "no", "off"}
|
||||
|
||||
|
||||
def parse_bool(value: Any, default: bool = False) -> bool:
|
||||
"""Coerce *value* to bool safely. Returns *default* for unrecognised input.
|
||||
|
||||
Recognises True/False, ``"true"``/``"false"``, ``"1"``/``"0"``,
|
||||
``"yes"``/``"no"``, ``"on"``/``"off"`` (case-insensitive). Strings
|
||||
outside these sets return *default* instead of silently being truthy.
|
||||
"""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
s = str(value).strip().lower()
|
||||
if s in _TRUE_VALUES:
|
||||
return True
|
||||
if s in _FALSE_VALUES:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
# ── L-8: trusted-host check ──────────────────────────────────────────────
|
||||
import ipaddress # noqa: E402
|
||||
|
||||
|
||||
def _is_trusted_host(host: str) -> bool:
|
||||
"""Return True if *host* (the ``Host`` header value) is a loopback /
|
||||
private address that this local-only deployment should trust."""
|
||||
hostname = host.split("@")[-1].split(":")[0] # strip auth / port
|
||||
if not hostname:
|
||||
return False
|
||||
if hostname in {"localhost", "127.0.0.1", "::1"}:
|
||||
return True
|
||||
try:
|
||||
addr = ipaddress.ip_address(hostname)
|
||||
return addr.is_loopback or addr.is_private or addr.is_link_local
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
# ── M-1: sensitive file / media extension allowlists ──────────────────────
|
||||
_SENSITIVE_FILE_SUFFIXES = frozenset({
|
||||
".db", ".session", ".db-wal", ".db-shm", ".db-journal",
|
||||
})
|
||||
_SENSITIVE_FILE_NAMES = frozenset({"state.json"})
|
||||
|
||||
# How long a generated QR login token/image is considered valid before it is
|
||||
# dropped from the auth snapshot (seconds). L-5.
|
||||
QR_TTL_SECONDS = 60
|
||||
|
||||
_MEDIA_FILE_EXTENSIONS = frozenset({
|
||||
# images
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg", ".ico",
|
||||
# video
|
||||
".mp4", ".webm", ".mov", ".m4v", ".avi",
|
||||
# audio
|
||||
".mp3", ".ogg", ".wav", ".m4a", ".flac",
|
||||
# documents / generic downloaded binary
|
||||
".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
|
||||
".txt", ".csv", ".json", ".xml", ".bin",
|
||||
# archives / other Telegram document types
|
||||
".zip", ".rar", ".7z", ".apk", ".epub", ".tar", ".gz", ".bz2", ".xz",
|
||||
".odt", ".ods", ".odp",
|
||||
})
|
||||
|
||||
|
||||
def _sweep_auth_attempts(now: float) -> None:
|
||||
"""Evict expired lockout/cooldown entries when the dict grows too large."""
|
||||
@@ -589,6 +657,23 @@ class JobRunner:
|
||||
len(still_running),
|
||||
timeout,
|
||||
)
|
||||
# M-7: The worker exits once _shutdown_flag is set, leaving any yet-to-be
|
||||
# processed queued jobs stranded as "queued". Drain the queue and mark
|
||||
# each leftover as failed so their event streams / polling terminate.
|
||||
drained = 0
|
||||
while True:
|
||||
try:
|
||||
job = self.queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self.queue.task_done()
|
||||
with self.lock:
|
||||
job.status = "failed"
|
||||
job.error = "Server shutting down; job was cancelled"
|
||||
job.finished_at = utc_now_iso()
|
||||
drained += 1
|
||||
if drained:
|
||||
logger.info("Failed %d queued job(s) during shutdown.", drained)
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._shutdown_flag:
|
||||
@@ -653,6 +738,7 @@ class TelegramAuthManager:
|
||||
"details": "",
|
||||
"qr_url": None,
|
||||
"qr_image": None,
|
||||
"qr_created_at": None,
|
||||
"phone": None,
|
||||
"phone_code_hash": None,
|
||||
"qr_login": None,
|
||||
@@ -713,6 +799,16 @@ class TelegramAuthManager:
|
||||
with self.lock:
|
||||
data = self._get_auth_data(account_id)
|
||||
snapshot = dict(data)
|
||||
# L-5: expire a QR login token that has not been scanned within
|
||||
# QR_TTL_SECONDS so it cannot live forever in the auth snapshot.
|
||||
qr_created = data.get("qr_created_at")
|
||||
if qr_created and (time.time() - float(qr_created)) > QR_TTL_SECONDS:
|
||||
if data.get("qr_url") or data.get("qr_image"):
|
||||
data["qr_url"] = None
|
||||
data["qr_image"] = None
|
||||
data["qr_created_at"] = None
|
||||
snapshot = dict(data)
|
||||
snapshot["qr_expired"] = True
|
||||
snapshot.pop("qr_login", None)
|
||||
snapshot.pop("qr_wait_task", None)
|
||||
snapshot.pop("phone_code_hash", None)
|
||||
@@ -768,6 +864,7 @@ class TelegramAuthManager:
|
||||
details="Scan the QR code in Telegram: Settings -> Devices -> Scan QR.",
|
||||
qr_url=qr_url,
|
||||
qr_image=self._make_qr_image(qr_url),
|
||||
qr_created_at=time.time(),
|
||||
)
|
||||
with self.lock:
|
||||
data = self._get_auth_data(account_id)
|
||||
@@ -792,6 +889,7 @@ class TelegramAuthManager:
|
||||
details="Telegram session is authorized.",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
qr_created_at=None,
|
||||
phone=None,
|
||||
)
|
||||
except SessionPasswordNeededError:
|
||||
@@ -809,6 +907,7 @@ class TelegramAuthManager:
|
||||
details=f"QR login failed: {exc}",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
qr_created_at=None,
|
||||
)
|
||||
|
||||
def start_qr_login(self, account_id: str) -> Dict[str, Any]:
|
||||
@@ -859,6 +958,7 @@ class TelegramAuthManager:
|
||||
details="Telegram session is authorized.",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
qr_created_at=None,
|
||||
)
|
||||
except SessionPasswordNeededError:
|
||||
self._set_state(
|
||||
@@ -882,6 +982,7 @@ class TelegramAuthManager:
|
||||
details="Telegram session is authorized.",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
qr_created_at=None,
|
||||
)
|
||||
return self.auth_state(account_id)
|
||||
|
||||
@@ -937,6 +1038,10 @@ class PerAccountContinuousScrapeManager:
|
||||
self.lock = threading.RLock()
|
||||
self.thread: Optional[threading.Thread] = None
|
||||
self.stop_event = threading.Event()
|
||||
# F-4: set on remove_account when its thread could not be joined in
|
||||
# time; the manager is kept as a tombstone so it is reused (after the
|
||||
# drain) instead of spawning a duplicate worker on account re-add.
|
||||
self._removing = False
|
||||
self.config: Dict[str, Any] = self._load_config()
|
||||
self.status: Dict[str, Any] = {
|
||||
"running": False,
|
||||
@@ -1053,9 +1158,34 @@ class PerAccountContinuousScrapeManager:
|
||||
|
||||
def start(self) -> None:
|
||||
with self.lock:
|
||||
if self.status["running"]:
|
||||
self._log("Continuous scraping is already running.", "warn")
|
||||
thread = self.thread
|
||||
if thread is not None and thread.is_alive():
|
||||
if self.stop_event.is_set():
|
||||
# F-3: a stop was requested and the old thread is still
|
||||
# draining (its finally has not yet flipped running=False).
|
||||
# Wait for it to exit before starting a fresh one so the
|
||||
# account does not end up enabled=True with a dead thread.
|
||||
self._log(
|
||||
"Previous scrape thread is stopping; waiting before restart.",
|
||||
"warn",
|
||||
)
|
||||
else:
|
||||
# A live thread is genuinely running — do NOT spawn a
|
||||
# duplicate.
|
||||
self._log("Continuous scraping is already running.", "warn")
|
||||
return
|
||||
if thread is not None and thread.is_alive():
|
||||
# Join outside the lock (bounded) so the draining thread can mark
|
||||
# itself finished. If it does not exit in time, refuse to start a
|
||||
# duplicate worker rather than risk overlapping writes.
|
||||
thread.join(timeout=5.0)
|
||||
if thread.is_alive():
|
||||
self._log(
|
||||
"Previous scrape thread still stopping; cannot restart yet.",
|
||||
"error",
|
||||
)
|
||||
return
|
||||
with self.lock:
|
||||
self.stop_event.clear()
|
||||
self.status["running"] = True
|
||||
self.status["last_started_at"] = utc_now_iso()
|
||||
@@ -1180,9 +1310,18 @@ class ContinuousScrapeOrchestrator:
|
||||
|
||||
def _get_or_create(self, account_id: str) -> PerAccountContinuousScrapeManager:
|
||||
with self.lock:
|
||||
if account_id not in self.managers:
|
||||
self.managers[account_id] = PerAccountContinuousScrapeManager(account_id)
|
||||
return self.managers[account_id]
|
||||
mgr = self.managers.get(account_id)
|
||||
if mgr is None:
|
||||
mgr = PerAccountContinuousScrapeManager(account_id)
|
||||
self.managers[account_id] = mgr
|
||||
elif mgr._removing:
|
||||
# F-4: the manager was left as a tombstone because its worker
|
||||
# could not be joined during remove_account. The account is
|
||||
# being re-added / accessed, so clear the tombstone and reuse
|
||||
# the manager (start() waits for the old thread to drain before
|
||||
# spawning a fresh one, avoiding duplicate workers).
|
||||
mgr._removing = False
|
||||
return mgr
|
||||
|
||||
def start_account(self, account_id: str) -> None:
|
||||
mgr = self._get_or_create(account_id)
|
||||
@@ -1240,18 +1379,24 @@ class ContinuousScrapeOrchestrator:
|
||||
mgr.stop()
|
||||
# Wait for the scrape thread to actually stop before the caller
|
||||
# deletes the account directory / session files, so rmtree does not
|
||||
# race with a writer mid-iteration. If the thread is still running
|
||||
# (e.g. mid-scrape) after the timeout we proceed best-effort and
|
||||
# log a warning.
|
||||
if not mgr.join(timeout=REMOVE_ACCOUNT_JOIN_TIMEOUT):
|
||||
# race with a writer mid-iteration.
|
||||
if mgr.join(timeout=REMOVE_ACCOUNT_JOIN_TIMEOUT):
|
||||
# Thread finished: safe to drop the manager.
|
||||
with self.lock:
|
||||
self.managers.pop(account_id, None)
|
||||
else:
|
||||
# F-4: the thread is still draining. Do NOT pop the manager —
|
||||
# leave it as a tombstone (marked removing) so a later re-add of
|
||||
# the same account id reuses it / waits for the drain instead of
|
||||
# spawning a duplicate worker that writes the same dirs.
|
||||
logger.warning(
|
||||
"Continuous scrape thread for account %r still running after "
|
||||
"%.1fs; removing account data best-effort",
|
||||
"%.1fs; keeping manager as tombstone",
|
||||
account_id,
|
||||
REMOVE_ACCOUNT_JOIN_TIMEOUT,
|
||||
)
|
||||
with self.lock:
|
||||
self.managers.pop(account_id, None)
|
||||
with self.lock:
|
||||
mgr._removing = True
|
||||
|
||||
def stop_all(self) -> None:
|
||||
with self.lock:
|
||||
@@ -1336,18 +1481,31 @@ def auth_status() -> Dict[str, Any]:
|
||||
return auth_status_for(account_id=None)
|
||||
|
||||
|
||||
def dashboard_payload(job_runner: JobRunner) -> Dict[str, Any]:
|
||||
state = load_state()
|
||||
channels = list_channels_snapshot()
|
||||
def dashboard_payload(job_runner: JobRunner, account_id: Optional[str] = None) -> Dict[str, Any]:
|
||||
if account_id:
|
||||
acc_state = load_account(DATA_DIR, account_id)
|
||||
state = {
|
||||
"scrape_media": bool(acc_state.get("scrape_media", True)),
|
||||
"channel_count": len(acc_state.get("channels", {})),
|
||||
"forwarding_rules": acc_state.get("forwarding_rules", []),
|
||||
}
|
||||
channels = list_channels_snapshot(account_id)
|
||||
auth = auth_status_for(account_id)
|
||||
jobs = job_runner.recent_jobs(account_id=account_id)
|
||||
else:
|
||||
state = load_state()
|
||||
channels = list_channels_snapshot()
|
||||
auth = auth_status()
|
||||
jobs = job_runner.recent_jobs()
|
||||
return {
|
||||
"state": {
|
||||
"scrape_media": bool(state.get("scrape_media", True)),
|
||||
"channel_count": len(state.get("channels", {})),
|
||||
"forwarding_rules": state.get("forwarding_rules", []),
|
||||
},
|
||||
"auth": auth_status(),
|
||||
"auth": auth,
|
||||
"channels": channels,
|
||||
"jobs": job_runner.recent_jobs(),
|
||||
"jobs": jobs,
|
||||
}
|
||||
|
||||
|
||||
@@ -2050,7 +2208,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
if path == "/openapi.json":
|
||||
return self.send_json(openapi_payload())
|
||||
if path == "/api/dashboard":
|
||||
return self.send_json(dashboard_payload(self.app.job_runner))
|
||||
return self.send_json(
|
||||
dashboard_payload(self.app.job_runner, self.app.legacy_account_id)
|
||||
)
|
||||
if path == "/api/auth":
|
||||
if self.app.legacy_account_id:
|
||||
return self.send_json(self.app.auth_manager.auth_state(self.app.legacy_account_id))
|
||||
@@ -2094,27 +2254,34 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found")
|
||||
return self.send_json(job)
|
||||
if path == "/api/channels":
|
||||
return self.send_json(list_channels_snapshot())
|
||||
return self.send_json(list_channels_snapshot(self.app.legacy_account_id))
|
||||
if path.startswith("/api/channels/") and path.endswith("/messages"):
|
||||
parts = path.split("/")
|
||||
try:
|
||||
channel_id = normalize_channel_id(urllib.parse.unquote(parts[3]))
|
||||
except ValueError as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
|
||||
try:
|
||||
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
|
||||
except (TypeError, ValueError):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'limit' query parameter")
|
||||
before = query.get("before")
|
||||
before_message_id = int(before[0]) if before else None
|
||||
try:
|
||||
before_message_id = int(before[0]) if before else None
|
||||
except (TypeError, ValueError):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'before' query parameter")
|
||||
search = (query.get("search") or query.get("q") or [""])[0].strip()
|
||||
account_id = self.app.legacy_account_id
|
||||
payload = {
|
||||
"channel_id": channel_id,
|
||||
"messages": load_messages(
|
||||
None, channel_id, limit=limit, before_message_id=before_message_id,
|
||||
account_id, channel_id, limit=limit, before_message_id=before_message_id,
|
||||
search=search or None
|
||||
),
|
||||
"channel": next(
|
||||
(
|
||||
item
|
||||
for item in list_channels_snapshot()
|
||||
for item in list_channels_snapshot(account_id)
|
||||
if item["channel_id"] == channel_id
|
||||
),
|
||||
None,
|
||||
@@ -2258,9 +2425,15 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
def _handle_get_account_channel_messages(
|
||||
self, account_id: str, channel_id: str, query: Dict[str, List[str]]
|
||||
) -> None:
|
||||
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
|
||||
try:
|
||||
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
|
||||
except (TypeError, ValueError):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'limit' query parameter")
|
||||
before = query.get("before")
|
||||
before_message_id = int(before[0]) if before else None
|
||||
try:
|
||||
before_message_id = int(before[0]) if before else None
|
||||
except (TypeError, ValueError):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Invalid 'before' query parameter")
|
||||
search = (query.get("search") or query.get("q") or [""])[0].strip()
|
||||
payload = {
|
||||
"channel_id": channel_id,
|
||||
@@ -2316,6 +2489,12 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
or (path.startswith("/api/channels/") and path.endswith("/messages"))
|
||||
or path.startswith("/api/accounts/")
|
||||
):
|
||||
# M-4: for the job-events route, mirror GET's behavior and verify
|
||||
# the job actually exists before responding with 200.
|
||||
if path.startswith("/api/jobs/") and path.endswith("/events"):
|
||||
job_id = path.split("/")[-2] if path.endswith("/events") else None
|
||||
if job_id and not self.app.job_runner.get_job(job_id):
|
||||
return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found")
|
||||
self.send_response(HTTPStatus.OK)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.end_headers()
|
||||
@@ -2339,22 +2518,23 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
|
||||
# ── Legacy endpoints ────────────────────────────────────────────
|
||||
if path == "/api/settings/media":
|
||||
value = bool(body.get("value"))
|
||||
value = parse_bool(body.get("value"))
|
||||
if self.app.legacy_account_id:
|
||||
return self._handle_account_settings_media(self.app.legacy_account_id, value)
|
||||
job = self.app.job_runner.create_job(
|
||||
"set_scrape_media",
|
||||
"Update media scraping setting",
|
||||
{"value": value},
|
||||
)
|
||||
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
|
||||
# No legacy account: persist to the (global) legacy store directly.
|
||||
# The ScraperJobService set_scrape_media job is a passthrough, so
|
||||
# enqueueing it alone would silently drop the setting.
|
||||
def _media_mutate(global_state: Dict[str, Any]) -> None:
|
||||
global_state["scrape_media"] = value
|
||||
STATE_STORE.update(_media_mutate)
|
||||
return self.send_json({"ok": True, "scrape_media": value})
|
||||
|
||||
if path == "/api/continuous":
|
||||
try:
|
||||
enabled = bool(body.get("enabled"))
|
||||
enabled = parse_bool(body.get("enabled"))
|
||||
interval_minutes = int(body.get("interval_minutes", 1))
|
||||
channels, dropped = clean_continuous_channels(body.get("channels", []))
|
||||
run_all_tracked = bool(body.get("run_all_tracked", True))
|
||||
run_all_tracked = parse_bool(body.get("run_all_tracked", True), default=True)
|
||||
if self.app.legacy_account_id:
|
||||
payload = self.app.continuous_orchestrator.update_for(
|
||||
account_id=self.app.legacy_account_id,
|
||||
@@ -2366,7 +2546,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
payload = {"config": {}, "status": {"running": False, "logs": [], "log_entries": []}}
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(
|
||||
exc, fallback="Failed to update continuous scraping"
|
||||
)
|
||||
if isinstance(payload, dict):
|
||||
payload["dropped_invalid"] = dropped
|
||||
return self.send_json(payload)
|
||||
@@ -2383,7 +2565,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
save_state(state)
|
||||
payload = self._legacy_auth_snapshot()
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(
|
||||
exc, fallback="Failed to save API credentials"
|
||||
)
|
||||
return self.send_json(payload)
|
||||
|
||||
if path == "/api/auth/qr/start":
|
||||
@@ -2405,12 +2589,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
if self.app.legacy_account_id:
|
||||
return self._handle_account_channel_add(self.app.legacy_account_id, channel_id, body.get("name"))
|
||||
state = load_state()
|
||||
if channel_id not in state["channels"]:
|
||||
state["channels"][channel_id] = 0
|
||||
if body.get("name"):
|
||||
state.setdefault("channel_names", {})[channel_id] = str(body["name"]).strip()
|
||||
save_state(state)
|
||||
# L-6: use the atomic mutator to avoid lost-update races.
|
||||
def _add_mutate(state: Dict[str, Any]) -> None:
|
||||
if channel_id not in state.get("channels", {}):
|
||||
state.setdefault("channels", {})[channel_id] = 0
|
||||
if body.get("name"):
|
||||
state.setdefault("channel_names", {})[channel_id] = str(body["name"]).strip()
|
||||
STATE_STORE.update(_add_mutate)
|
||||
return self.send_json({"ok": True, "channel_id": channel_id})
|
||||
|
||||
if path == "/api/channels/remove":
|
||||
@@ -2420,11 +2605,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
if self.app.legacy_account_id:
|
||||
return self._handle_account_channel_remove(self.app.legacy_account_id, channel_id)
|
||||
state = load_state()
|
||||
existed = channel_id in state.get("channels", {})
|
||||
state.get("channels", {}).pop(channel_id, None)
|
||||
save_state(state)
|
||||
return self.send_json({"ok": existed, "channel_id": channel_id})
|
||||
# L-6: use the atomic mutator to avoid lost-update races.
|
||||
existed = [False]
|
||||
def _remove_mutate(state: Dict[str, Any]) -> None:
|
||||
chans = state.get("channels", {})
|
||||
if channel_id in chans:
|
||||
existed[0] = True
|
||||
del chans[channel_id]
|
||||
state.get("channel_names", {}).pop(channel_id, None)
|
||||
STATE_STORE.update(_remove_mutate)
|
||||
return self.send_json({"ok": existed[0], "channel_id": channel_id})
|
||||
|
||||
if path == "/api/jobs/scrape":
|
||||
channel_id = body.get("channel_id")
|
||||
@@ -2581,19 +2771,30 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
if not isinstance(state, dict):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "state object is required")
|
||||
|
||||
# F-1: normalise the imported continuous channels and drop invalid
|
||||
# entries (path traversal / control chars) before persisting.
|
||||
cs = state.get("continuous_scraping")
|
||||
cs = cs if isinstance(cs, dict) else {}
|
||||
cs_channels, _dropped = clean_continuous_channels(cs.get("channels", []))
|
||||
|
||||
try:
|
||||
interval_minutes = int(cs.get("interval_minutes", 1) or 1)
|
||||
except (TypeError, ValueError):
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "interval_minutes must be a valid integer")
|
||||
|
||||
imported_state = {
|
||||
"label": str(body.get("label") or state.get("label") or account_id).strip(),
|
||||
"api_id": state.get("api_id"),
|
||||
"api_hash": state.get("api_hash"),
|
||||
"channels": self._clean_imported_channels(state.get("channels")),
|
||||
"channel_names": self._clean_imported_channel_names(state.get("channel_names")),
|
||||
"scrape_media": bool(state.get("scrape_media", True)),
|
||||
"scrape_media": parse_bool(state.get("scrape_media", True), default=True),
|
||||
"forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [],
|
||||
"continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else {
|
||||
"enabled": False,
|
||||
"interval_minutes": 1,
|
||||
"channels": [],
|
||||
"run_all_tracked": True,
|
||||
"continuous_scraping": {
|
||||
"enabled": parse_bool(cs.get("enabled", False)),
|
||||
"interval_minutes": interval_minutes,
|
||||
"channels": cs_channels,
|
||||
"run_all_tracked": parse_bool(cs.get("run_all_tracked", True), default=True),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2645,7 +2846,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
if sub == ["jobs", "refresh-dialogs"]:
|
||||
return self._handle_account_job_refresh_dialogs(account_id)
|
||||
if sub == ["settings", "media"]:
|
||||
return self._handle_account_settings_media(account_id, bool(body.get("value")))
|
||||
return self._handle_account_settings_media(account_id, parse_bool(body.get("value")))
|
||||
if sub == ["continuous"]:
|
||||
return self._handle_account_continuous(account_id, body)
|
||||
return self.send_error_json(HTTPStatus.NOT_FOUND, "Not found")
|
||||
@@ -2664,14 +2865,14 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
payload = self.app.auth_manager.save_credentials(account_id, int(api_id), api_hash)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(exc, fallback="Failed to save API credentials")
|
||||
return self.send_json(payload)
|
||||
|
||||
def _handle_post_account_auth_qr_start(self, account_id: str) -> None:
|
||||
try:
|
||||
payload = self.app.auth_manager.start_qr_login(account_id)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(exc, fallback="Failed to start QR login")
|
||||
return self.send_json(payload)
|
||||
|
||||
def _handle_post_account_auth_phone_request(self, account_id: str, body: Dict[str, Any]) -> None:
|
||||
@@ -2693,7 +2894,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
payload = self.app.auth_manager.request_phone_code(account_id, phone)
|
||||
except Exception as exc:
|
||||
_record_auth_failure(ip, account_id)
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(exc, fallback="Failed to request phone code")
|
||||
_record_auth_code_request(ip, account_id)
|
||||
return self.send_json(payload)
|
||||
|
||||
@@ -2711,7 +2912,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
payload = self.app.auth_manager.submit_phone_code(account_id, code)
|
||||
except Exception as exc:
|
||||
_record_auth_failure(ip, account_id)
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(exc, fallback="Failed to submit phone code")
|
||||
_record_auth_success(ip, account_id)
|
||||
return self.send_json(payload)
|
||||
|
||||
@@ -2729,7 +2930,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
payload = self.app.auth_manager.submit_password(account_id, password)
|
||||
except Exception as exc:
|
||||
_record_auth_failure(ip, account_id)
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(exc, fallback="Failed to submit password")
|
||||
_record_auth_success(ip, account_id)
|
||||
return self.send_json(payload)
|
||||
|
||||
@@ -2875,10 +3076,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def _handle_account_continuous(self, account_id: str, body: Dict[str, Any]) -> None:
|
||||
try:
|
||||
enabled = bool(body.get("enabled"))
|
||||
enabled = parse_bool(body.get("enabled"))
|
||||
interval_minutes = int(body.get("interval_minutes", 1))
|
||||
channels, dropped = clean_continuous_channels(body.get("channels", []))
|
||||
run_all_tracked = bool(body.get("run_all_tracked", True))
|
||||
run_all_tracked = parse_bool(body.get("run_all_tracked", True), default=True)
|
||||
payload = self.app.continuous_orchestrator.update_for(
|
||||
account_id=account_id,
|
||||
enabled=enabled,
|
||||
@@ -2887,7 +3088,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
run_all_tracked=run_all_tracked,
|
||||
)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_bad_request_from_exc(exc, fallback="Failed to update continuous scraping")
|
||||
payload["dropped_invalid"] = dropped
|
||||
return self.send_json(payload)
|
||||
|
||||
@@ -2983,6 +3184,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
if not origin_host or (host and origin_host != host):
|
||||
self.send_error_json(HTTPStatus.FORBIDDEN, "Cross-origin request rejected")
|
||||
return False
|
||||
# L-8: When an Origin is present, guard against DNS rebinding: the
|
||||
# Host must resolve to a local/private address, otherwise a remote
|
||||
# attacker domain could make both Host == Origin pass. Requests
|
||||
# without an Origin (curl, plain LAN tools) are unaffected.
|
||||
if host and not _is_trusted_host(host):
|
||||
self.send_error_json(HTTPStatus.FORBIDDEN, "Untrusted host")
|
||||
return False
|
||||
sec_fetch_site = self.headers.get("Sec-Fetch-Site", "").strip().lower()
|
||||
if sec_fetch_site and sec_fetch_site not in {"same-origin", "same-site", "none"}:
|
||||
self.send_error_json(HTTPStatus.FORBIDDEN, "Cross-origin request rejected")
|
||||
@@ -3007,6 +3215,22 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
file_path.relative_to(DATA_DIR.resolve())
|
||||
except ValueError:
|
||||
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
|
||||
|
||||
# M-1: Never serve sensitive state / session / db files. Normal
|
||||
# media documents live under DATA_DIR with a media-file extension;
|
||||
# anything else (state.json, *.db, *.session, sqlite sidecars) is
|
||||
# denied regardless of its location under DATA_DIR.
|
||||
file_suffix = file_path.suffix.lower()
|
||||
file_name = file_path.name.lower()
|
||||
if file_name in _SENSITIVE_FILE_NAMES or file_suffix in _SENSITIVE_FILE_SUFFIXES:
|
||||
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
|
||||
# Only allow registered media/document extensions. This keeps the
|
||||
# viewer's image/video/audio/document URLs working while blocking
|
||||
# arbitrary file reads (e.g. /media/../state.json is already blocked
|
||||
# by containment, and any other extension is not a media asset).
|
||||
if file_suffix not in _MEDIA_FILE_EXTENSIONS:
|
||||
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
|
||||
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
return self.send_error_json(HTTPStatus.NOT_FOUND, "Media file not found")
|
||||
mime_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
|
||||
@@ -3018,6 +3242,11 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
last_modified = stat.st_mtime
|
||||
|
||||
range_header = self.headers.get("Range", "").strip()
|
||||
# M-8: for an empty file there is nothing to range-serve; ignore the
|
||||
# Range header entirely and send the full (empty) 200 response so
|
||||
# clients don't see a spurious 416 for `bytes=0-0` on size-0 files.
|
||||
if file_size == 0:
|
||||
range_header = ""
|
||||
if range_header.startswith("bytes="):
|
||||
try:
|
||||
start, end = self._parse_range(range_header, file_size)
|
||||
@@ -3034,6 +3263,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
)
|
||||
self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
|
||||
self.send_header("Content-Length", str(content_length))
|
||||
self._send_security_headers()
|
||||
if "text/html" in content_type:
|
||||
self.send_header("Content-Security-Policy", self._CSP)
|
||||
self.end_headers()
|
||||
if not head_only:
|
||||
self._write_file_range(file_path, start, content_length)
|
||||
@@ -3046,24 +3278,42 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
|
||||
)
|
||||
self.send_header("Content-Length", str(file_size))
|
||||
self._send_security_headers()
|
||||
if "text/html" in content_type:
|
||||
self.send_header("Content-Security-Policy", self._CSP)
|
||||
self.end_headers()
|
||||
if not head_only:
|
||||
self._write_file_range(file_path, 0, file_size)
|
||||
|
||||
def _parse_range(self, range_header: str, file_size: int) -> tuple[int, int]:
|
||||
range_val = range_header.removeprefix("bytes=").strip()
|
||||
if "-" not in range_val:
|
||||
raise ValueError("missing dash")
|
||||
parts = range_val.split("-", 1)
|
||||
if parts[0] == "":
|
||||
end = int(parts[1])
|
||||
start = max(0, file_size - end)
|
||||
else:
|
||||
start = int(parts[0])
|
||||
end = int(parts[1]) if parts[1] else file_size - 1
|
||||
if start < 0 or start >= file_size or end >= file_size or start > end:
|
||||
raise ValueError("out of bounds")
|
||||
return start, end
|
||||
"""Parse a single ``bytes=start-end`` range spec.
|
||||
|
||||
Raises ``ValueError`` (cleanly caught by the caller → 416) for any
|
||||
malformed / multi-range / out-of-bounds input. Never lets an
|
||||
exception other than ValueError escape.
|
||||
"""
|
||||
try:
|
||||
range_val = range_header.removeprefix("bytes=").strip()
|
||||
# Multi-range requests ("bytes=0-1,5-6") are not supported here.
|
||||
if "," in range_val:
|
||||
raise ValueError("multi-range not supported")
|
||||
if "-" not in range_val:
|
||||
raise ValueError("missing dash")
|
||||
parts = range_val.split("-", 1)
|
||||
if parts[0] == "":
|
||||
end = int(parts[1])
|
||||
start = max(0, file_size - end)
|
||||
else:
|
||||
start = int(parts[0])
|
||||
end = int(parts[1]) if parts[1] else file_size - 1
|
||||
if start < 0 or start >= file_size or end >= file_size or start > end:
|
||||
raise ValueError("out of bounds")
|
||||
return start, end
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.debug("Unexpected range parse error: %s", exc)
|
||||
raise ValueError("invalid range") from exc
|
||||
|
||||
def _write_file_range(self, file_path: Path, offset: int, length: int) -> None:
|
||||
chunk_size = 65536
|
||||
@@ -3077,22 +3327,42 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
|
||||
# ── L-4: Security headers ─────────────────────────────────────────────
|
||||
_CSP = "default-src 'self'; img-src 'self' data:; style-src 'self'"
|
||||
|
||||
def _send_security_headers(self) -> None:
|
||||
"""Emit standard hardening headers on every HTTP response."""
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("X-Frame-Options", "DENY")
|
||||
self.send_header("Referrer-Policy", "no-referrer")
|
||||
|
||||
def send_json(self, payload: Any, status: HTTPStatus = HTTPStatus.OK) -> None:
|
||||
raw = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self._send_security_headers()
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def send_error_json(self, status: HTTPStatus, message: str) -> None:
|
||||
self.send_json({"error": message, "status": int(status)}, status=status)
|
||||
|
||||
def send_bad_request_from_exc(
|
||||
self, exc: Exception, fallback: str = "Bad request"
|
||||
) -> None:
|
||||
"""Send a 400 JSON error without leaking filesystem paths or other
|
||||
server internals. The caught exception is logged server-side and the
|
||||
client receives only *fallback*."""
|
||||
logger.exception("Request failed; client sees '%s'", fallback)
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, fallback)
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
safe_path = self.path.split("?", 1)[0]
|
||||
logger.info(
|
||||
"%s %s — %s",
|
||||
self.command,
|
||||
self.path,
|
||||
safe_path,
|
||||
args[1] if len(args) > 1 else "-",
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user