forust/telegram-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Activity

fix(server): harden auth, SSE, state, scraping

Browse Source 
- Fix SSE streams not terminating on successful jobs (C-2)
- Anchor data/session paths to BASE_DIR instead of CWD (C-3)
- Guard TelegramAuthManager state with RLock (H-1)
- Replace millisecond job ids with uuid4 (H-2)
- Always redact api_id/api_hash on export, drop include_secrets (H-3)
- Enforce JSON content-type + same-origin on mutating requests (H-4)
- Rate-limit auth attempts and phone-code requests (H-5)
- Deep-copy StateStore.load() on all paths (H-6)
- Cap FloodWait retries in forward_message (H-7)
- De-duplicate forwarding handler registration (H-8)
- Validate continuous channels at ingest, join scrape thread on account
  removal, fix refresh_config status under lock, cap SSE streams and
  JSON body size (M-5, M-6, M-17)
- Add regression tests (33 passing) and REVIEW.md
This commit is contained in:
mr-forust
2026-09-07 11:54:13 +02:00
parent 59824940c6
commit a2468a2a2c
8 changed files with 973 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui_server.py
+478 -163
View File
@@ -14,12 +14,13 @@ import threading
import time
import traceback
import urllib.parse
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
import qrcode
import qrcode.image.svg
@@ -54,6 +55,116 @@ SCRAPER_JOBS = ScraperJobService(STATE_STORE)
logger = logging.getLogger(__name__)
# Job statuses that are terminal: once a job reaches one of these states the
# job is considered finished and event streams / polling should stop.
TERMINAL_JOB_STATUSES = {"done", "failed", "completed"}
# Maximum number of concurrently open SSE event streams. Each stream pins a
# handler thread for up to 30 minutes, so cap them and close the oldest.
MAX_EVENT_STREAMS = 10
# How long ContinuousScrapeOrchestrator.remove_account waits for a scrape
# worker thread to exit before deleting account data best-effort (seconds).
REMOVE_ACCOUNT_JOIN_TIMEOUT = 20.0
# Sentinel returned by read_json_body() when a request carries a body with a
# non-JSON Content-Type - maps to a 415 Unsupported Media Type response.
_JSON_CONTENT_TYPE_REJECTED = object()
# Sentinel returned by read_json_body() when the declared/actual body size
# exceeds MAX_JSON_BODY_BYTES - maps to a 413 Payload Too Large response.
_JSON_BODY_TOO_LARGE = object()
MAX_JSON_BODY_BYTES = 1_048_576 # 1 MB
# ── Auth throttling ─────────────────────────────────────────────────────
# Repeated failed submissions on the phone-code / 2FA-password endpoints are
# throttled per (client IP, account): AUTH_MAX_FAILED_ATTEMPTS failures lock
# the pair out for AUTH_LOCKOUT_SECONDS.
AUTH_MAX_FAILED_ATTEMPTS = 5
AUTH_LOCKOUT_SECONDS = 60
# Minimum interval (seconds) between successful phone-code REQUEST calls for
# the same (client IP, account) pair. Covers the success path so a LAN client
# cannot loop request_phone_code and flood the victim's phone with SMS.
AUTH_CODE_REQUEST_COOLDOWN_SECONDS = 30
# Cap the in-memory auth tracking dict; beyond this we sweep expired entries.
AUTH_ATTEMPTS_MAX_ENTRIES = 10_000
_auth_attempts: Dict[Tuple[str, str], Dict[str, Any]] = {}
_auth_attempts_lock = threading.Lock()
def _sweep_auth_attempts(now: float) -> None:
"""Evict expired lockout/cooldown entries when the dict grows too large."""
global _auth_attempts
if len(_auth_attempts) <= AUTH_ATTEMPTS_MAX_ENTRIES:
return
_auth_attempts = {
k: v
for k, v in _auth_attempts.items()
if (v.get("locked_until") or 0) > now or (v.get("cooldown_until") or 0) > now
}
def _check_auth_throttle(ip: str, account_id: str) -> bool:
"""Return True if the attempt is allowed, False if currently locked out."""
now = time.time()
with _auth_attempts_lock:
_sweep_auth_attempts(now)
entry = _auth_attempts.get((ip, account_id))
if not entry:
return True
locked_until = entry.get("locked_until")
if locked_until and now < locked_until:
return False
if locked_until and now >= locked_until:
# Lockout window expired - reset the counter.
_auth_attempts.pop((ip, account_id), None)
return True
def _check_auth_code_cooldown(ip: str, account_id: str) -> bool:
"""Return True if a code REQUEST is allowed, False if in cooldown."""
now = time.time()
with _auth_attempts_lock:
_sweep_auth_attempts(now)
entry = _auth_attempts.get((ip, account_id))
if not entry:
return True
cooldown_until = entry.get("cooldown_until")
if cooldown_until and now < cooldown_until:
return False
return True
def _record_auth_failure(ip: str, account_id: str) -> None:
now = time.time()
with _auth_attempts_lock:
_sweep_auth_attempts(now)
entry = _auth_attempts.setdefault(
(ip, account_id), {"failures": 0, "locked_until": None}
)
locked_until = entry.get("locked_until")
if locked_until is not None and now < locked_until:
return
entry["failures"] = entry.get("failures", 0) + 1
if entry["failures"] >= AUTH_MAX_FAILED_ATTEMPTS:
entry["locked_until"] = now + AUTH_LOCKOUT_SECONDS
entry["failures"] = 0
def _record_auth_success(ip: str, account_id: str) -> None:
with _auth_attempts_lock:
_auth_attempts.pop((ip, account_id), None)
def _record_auth_code_request(ip: str, account_id: str) -> None:
"""Record a successful phone-code request to start its cooldown."""
now = time.time()
with _auth_attempts_lock:
entry = _auth_attempts.setdefault(
(ip, account_id), {"failures": 0, "locked_until": None}
)
entry["cooldown_until"] = now + AUTH_CODE_REQUEST_COOLDOWN_SECONDS
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
@@ -112,11 +223,41 @@ def normalize_channel_id(value: Any) -> str:
return channel_id
def export_account_state(state: Dict[str, Any], include_secrets: bool = False) -> Dict[str, Any]:
def clean_continuous_channels(channels: Any) -> Tuple[List[str], List[str]]:
"""Validate/normalize a continuous-scrape channel list at ingest.
Returns ``(cleaned, dropped)`` where ``cleaned`` holds the normalized,
path-safe channel ids and ``dropped`` holds the raw values that were
rejected (invalid path characters, control chars, ``.``/``..``). The
dropped entries must NOT be persisted so they cannot become a path
traversal vector for any future direct read.
"""
cleaned: List[str] = []
dropped: List[str] = []
if not isinstance(channels, list):
return cleaned, dropped
for item in channels:
raw = str(item).strip()
try:
cleaned.append(normalize_channel_id(raw))
except ValueError:
dropped.append(raw)
return cleaned, dropped
def export_account_state(state: Dict[str, Any]) -> Dict[str, Any]:
"""Return a sanitized copy of an account state for export.
Credentials (api_hash, api_id) are always redacted - they are never
included in exports, only a presence flag is exposed.
"""
exported = dict(state)
if not include_secrets and "api_hash" in exported:
if "api_hash" in exported:
exported["api_hash_present"] = bool(exported.get("api_hash"))
exported["api_hash"] = None
if "api_id" in exported:
exported["api_id_present"] = bool(exported.get("api_id"))
exported["api_id"] = None
return exported
@@ -345,9 +486,33 @@ class JobRunner:
self.queue: "queue.Queue[Job]" = queue.Queue()
self.lock = threading.Lock()
self._shutdown_flag = False
# Active SSE event streams: stream_id -> started_at, plus any stream
# ids that have been revoked (told to close) because the cap was hit.
self._stream_started: Dict[str, float] = {}
self._stream_revoked: set = set()
self.worker = threading.Thread(target=self._run, daemon=True)
self.worker.start()
def register_event_stream(self) -> str:
"""Register an active SSE stream; revoke the oldest when over the cap."""
stream_id = uuid.uuid4().hex
with self.lock:
self._stream_started[stream_id] = time.time()
if len(self._stream_started) > MAX_EVENT_STREAMS:
oldest = min(self._stream_started, key=lambda sid: self._stream_started[sid])
self._stream_started.pop(oldest, None)
self._stream_revoked.add(oldest)
return stream_id
def unregister_event_stream(self, stream_id: str) -> None:
with self.lock:
self._stream_started.pop(stream_id, None)
self._stream_revoked.discard(stream_id)
def is_stream_revoked(self, stream_id: str) -> bool:
with self.lock:
return stream_id in self._stream_revoked
def create_job(self, job_type: str, title: str, payload: Dict[str, Any]) -> Job:
if self._shutdown_flag:
raise RuntimeError("Server is shutting down, cannot create new jobs")
@@ -366,7 +531,7 @@ class JobRunner:
+ f"[{datetime.now().strftime('%H:%M:%S')}] Reused existing active job for this account."
).strip()
return existing
job_id = f"job-{int(time.time() * 1000)}"
job_id = f"job-{uuid.uuid4().hex[:12]}"
job = Job(
job_id=job_id,
job_type=job_type,
@@ -459,7 +624,12 @@ class JobRunner:
class TelegramAuthManager:
def __init__(self) -> None:
self.lock = threading.Lock()
# Reentrant lock guarding self.auth_data / self.clients. These dicts
# are mutated from the async event-loop thread (coroutines) and read
# from HTTP handler threads, so every access must hold the lock.
# RLock allows nested acquisition from helpers (_get_auth_data /
# _set_state) called inside compound locked operations.
self.lock = threading.RLock()
self.loop = asyncio.new_event_loop()
self.thread = threading.Thread(target=self._run_loop, daemon=True)
self.thread.start()
@@ -475,26 +645,28 @@ class TelegramAuthManager:
return future.result()
def _get_auth_data(self, account_id: str) -> Dict[str, Any]:
if account_id not in self.auth_data:
self.auth_data[account_id] = {
"phase": "idle",
"status": "unknown",
"details": "",
"qr_url": None,
"qr_image": None,
"phone": None,
"phone_code_hash": None,
"qr_login": None,
"qr_wait_task": None,
"user_id": None,
"updated_at": utc_now_iso(),
}
return self.auth_data[account_id]
with self.lock:
if account_id not in self.auth_data:
self.auth_data[account_id] = {
"phase": "idle",
"status": "unknown",
"details": "",
"qr_url": None,
"qr_image": None,
"phone": None,
"phone_code_hash": None,
"qr_login": None,
"qr_wait_task": None,
"user_id": None,
"updated_at": utc_now_iso(),
}
return self.auth_data[account_id]
def _set_state(self, account_id: str, **updates: Any) -> None:
data = self._get_auth_data(account_id)
data.update(updates)
data["updated_at"] = utc_now_iso()
with self.lock:
data = self._get_auth_data(account_id)
data.update(updates)
data["updated_at"] = utc_now_iso()
def _make_qr_image(self, qr_url: str) -> str:
qr = qrcode.QRCode(border=1, box_size=8)
@@ -512,32 +684,39 @@ class TelegramAuthManager:
api_hash = acc_state.get("api_hash")
if not api_id or not api_hash:
raise RuntimeError("Save api_id and api_hash first for this account.")
if account_id not in self.clients or self.clients[account_id] is None:
_ensure_session_wal(account_session_path(SESSION_DIR, account_id))
self.clients[account_id] = TelegramClient(
account_session_path(SESSION_DIR, account_id),
api_id,
api_hash,
)
client = self.clients[account_id]
with self.lock:
if account_id not in self.clients or self.clients[account_id] is None:
_ensure_session_wal(account_session_path(SESSION_DIR, account_id))
self.clients[account_id] = TelegramClient(
account_session_path(SESSION_DIR, account_id),
api_id,
api_hash,
)
client = self.clients[account_id]
if not client.is_connected():
await client.connect()
data = self._get_auth_data(account_id)
if data.get("user_id") is None and await client.is_user_authorized():
with self.lock:
needs_user_id = self._get_auth_data(account_id).get("user_id") is None
if needs_user_id and await client.is_user_authorized():
try:
me = await client.get_me()
data["user_id"] = me.id
except Exception:
pass
me = None
if me is not None:
with self.lock:
data = self._get_auth_data(account_id)
if data.get("user_id") is None:
data["user_id"] = me.id
return client
def auth_state(self, account_id: str) -> Dict[str, Any]:
data = self._get_auth_data(account_id)
acc_state = load_account(DATA_DIR, account_id)
snapshot = dict(data)
with self.lock:
data = self._get_auth_data(account_id)
snapshot = dict(data)
snapshot.pop("qr_login", None)
snapshot.pop("qr_wait_task", None)
snapshot.pop("phone_code_hash", None)
acc_state = load_account(DATA_DIR, account_id)
snapshot["saved_credentials"] = {
"api_id": acc_state.get("api_id"),
"api_hash_present": bool(acc_state.get("api_hash")),
@@ -568,7 +747,6 @@ class TelegramAuthManager:
async def _start_qr_login(self, account_id: str) -> Dict[str, Any]:
client = await self._get_client(account_id)
data = self._get_auth_data(account_id)
if await client.is_user_authorized():
self._set_state(
account_id,
@@ -579,7 +757,10 @@ class TelegramAuthManager:
return self.auth_state(account_id)
qr_login = await client.qr_login()
qr_url = qr_login.url
data["qr_login"] = qr_login
with self.lock:
data = self._get_auth_data(account_id)
data["qr_login"] = qr_login
data["updated_at"] = utc_now_iso()
self._set_state(
account_id,
phase="qr_waiting",
@@ -588,14 +769,18 @@ class TelegramAuthManager:
qr_url=qr_url,
qr_image=self._make_qr_image(qr_url),
)
data["qr_wait_task"] = self.loop.create_task(
self._wait_for_qr_login(account_id)
)
with self.lock:
data = self._get_auth_data(account_id)
data["qr_wait_task"] = self.loop.create_task(
self._wait_for_qr_login(account_id)
)
data["updated_at"] = utc_now_iso()
return self.auth_state(account_id)
async def _wait_for_qr_login(self, account_id: str) -> None:
data = self._get_auth_data(account_id)
qr_login = data.get("qr_login")
with self.lock:
data = self._get_auth_data(account_id)
qr_login = data.get("qr_login")
if not qr_login:
return
try:
@@ -631,7 +816,6 @@ class TelegramAuthManager:
async def _request_phone_code(self, account_id: str, phone: str) -> Dict[str, Any]:
client = await self._get_client(account_id)
data = self._get_auth_data(account_id)
if await client.is_user_authorized():
self._set_state(
account_id,
@@ -641,14 +825,13 @@ class TelegramAuthManager:
)
return self.auth_state(account_id)
sent = await client.send_code_request(phone)
data["phone"] = phone
data["phone_code_hash"] = sent.phone_code_hash
self._set_state(
account_id,
phase="code_required",
status="code_required",
details=f"Code sent to {phone}. Enter it below.",
phone=phone,
phone_code_hash=sent.phone_code_hash,
)
return self.auth_state(account_id)
@@ -657,14 +840,17 @@ class TelegramAuthManager:
async def _submit_phone_code(self, account_id: str, code: str) -> Dict[str, Any]:
client = await self._get_client(account_id)
data = self._get_auth_data(account_id)
if not data.get("phone") or not data.get("phone_code_hash"):
with self.lock:
data = self._get_auth_data(account_id)
phone = data.get("phone")
phone_code_hash = data.get("phone_code_hash")
if not phone or not phone_code_hash:
raise RuntimeError("Request a phone code first.")
try:
await client.sign_in(
phone=data["phone"],
phone=phone,
code=code,
phone_code_hash=data["phone_code_hash"],
phone_code_hash=phone_code_hash,
)
self._set_state(
account_id,
@@ -703,8 +889,9 @@ class TelegramAuthManager:
return self._run(self._submit_password(account_id, password))
def delete_account(self, account_id: str) -> None:
if account_id in self.clients:
client = self.clients[account_id]
with self.lock:
client = self.clients.get(account_id)
if client is not None:
try:
future = asyncio.run_coroutine_threadsafe(
self._disconnect_client(client), self.loop
@@ -712,23 +899,27 @@ class TelegramAuthManager:
future.result(timeout=5)
except Exception:
pass
del self.clients[account_id]
self.auth_data.pop(account_id, None)
with self.lock:
self.clients.pop(account_id, None)
self.auth_data.pop(account_id, None)
async def _disconnect_client(self, client: TelegramClient) -> None:
if client and client.is_connected():
await client.disconnect()
def shutdown(self, timeout: float = 5.0) -> None:
with self.lock:
clients = list(self.clients.values())
async def _disconnect_all():
for client in self.clients.values():
for client in clients:
try:
if client and client.is_connected():
await client.disconnect()
except Exception:
pass
if self.clients:
if clients:
try:
future = asyncio.run_coroutine_threadsafe(_disconnect_all(), self.loop)
future.result(timeout=timeout)
@@ -799,13 +990,15 @@ class PerAccountContinuousScrapeManager:
if str(item).strip()
]
self.config["run_all_tracked"] = bool(disk_cfg.get("run_all_tracked", True))
# Sync running state with desired enabled state
# Sync running state with desired enabled state. We only request a
# stop here — the worker thread flips status["running"] to False in
# its finally block only once it actually exits, so the status flag
# never lies about a still-running scrape iteration.
if self.config["enabled"] and not self.status["running"]:
pass # don't auto-start — user must call start()
elif not self.config["enabled"] and self.status["running"]:
self._log("Continuous disabled via external state change, stopping.", "warn")
self.stop_event.set()
self.status["running"] = False
self._log("Config refreshed from disk.", "debug")
def _log(self, message: str, level: str = "debug") -> None:
@@ -848,7 +1041,10 @@ class PerAccountContinuousScrapeManager:
"run_all_tracked": bool(run_all_tracked),
}
self._save_config()
self.config = self._load_config()
# Re-read from disk under the same lock so config assignment is
# atomic w.r.t. refresh_config() (which also mutates config under
# the lock), avoiding a torn read / write race.
self.config = self._load_config()
if enabled:
self.start()
else:
@@ -864,17 +1060,36 @@ class PerAccountContinuousScrapeManager:
self.status["running"] = True
self.status["last_started_at"] = utc_now_iso()
self.status["last_error"] = None
self.thread = threading.Thread(target=self._run_loop, daemon=True)
self.thread = threading.Thread(target=self._run_loop, daemon=True)
self.thread.start()
self._log("Continuous scraping started.", "info")
def stop(self) -> None:
# Idempotent: only the first call logs. We deliberately do NOT set
# status["running"] = False here — the worker thread flips it once it
# has actually exited (see _run_loop's finally), so status reflects
# reality rather than intent.
already_set = self.stop_event.is_set()
self.stop_event.set()
with self.lock:
was_running = self.status["running"]
self.status["running"] = False
if was_running:
self._log("Continuous scraping stop requested.", "warn")
if not already_set:
with self.lock:
was_running = self.status["running"]
if was_running:
self._log("Continuous scraping stop requested.", "warn")
def join(self, timeout: float = 20.0) -> bool:
"""Wait up to ``timeout`` seconds for the worker thread to exit.
Safe to call when the thread was never started or is already dead.
Returns True if the thread finished within the timeout, False if it is
still running (e.g. mid-scrape) and the caller should proceed
best-effort.
"""
thread = self.thread
if thread is None or not thread.is_alive():
return True
thread.join(timeout=timeout)
return not thread.is_alive()
def _resolve_channels(self) -> List[str]:
acc_state = load_account(DATA_DIR, self.account_id)
@@ -887,66 +1102,73 @@ class PerAccountContinuousScrapeManager:
return [channel for channel in configured if channel in tracked]
def _run_loop(self) -> None:
while not self.stop_event.is_set():
# Refresh config from disk so channel / setting changes take effect
self.refresh_config()
try:
while not self.stop_event.is_set():
# Refresh config from disk so channel / setting changes take effect
self.refresh_config()
# Bail promptly if refresh/cancel requested the stop so join()
# usually returns quickly instead of waiting out a full scrape.
if self.stop_event.is_set():
break
# Check auth — don't iterate if account isn't authorized
auth_info = auth_status_for(self.account_id)
if auth_info.get("status") not in ("ready", "authorized"):
self._log(
f"Account not authorized (status={auth_info.get('status')}), "
"skipping iteration",
"warn",
)
sleep_seconds = max(5, 60)
# Check auth — don't iterate if account isn't authorized
auth_info = auth_status_for(self.account_id)
if auth_info.get("status") not in ("ready", "authorized"):
self._log(
f"Account not authorized (status={auth_info.get('status')}), "
"skipping iteration",
"warn",
)
sleep_seconds = max(5, 60)
interrupted = self.stop_event.wait(timeout=sleep_seconds)
if interrupted:
break
continue
channels = self._resolve_channels()
cfg = self.snapshot()["config"]
interval_minutes = cfg.get("interval_minutes", 1)
if not channels:
self._log("No channels configured for continuous scraping.", "warn")
else:
self._log(f"Starting iteration for {len(channels)} channel(s).", "info")
buffer = io.StringIO()
try:
with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer):
run_job("scrape_selected", {
"channels": channels,
"account_id": self.account_id,
})
output = buffer.getvalue().strip()
if output:
for line in output.splitlines():
self._log(line)
with self.lock:
self.status["last_iteration_at"] = utc_now_iso()
self.status["last_finished_at"] = utc_now_iso()
self.status["last_error"] = None
self._log("Iteration finished.", "success")
except Exception as exc:
output = buffer.getvalue().strip()
if output:
for line in output.splitlines():
self._log(line)
self._log(f"Iteration failed: {exc}", "error")
with self.lock:
self.status["last_error"] = str(exc)
sleep_seconds = max(5, interval_minutes * 60)
self._log(f"Sleeping for {interval_minutes} minute(s).", "debug")
interrupted = self.stop_event.wait(timeout=sleep_seconds)
if interrupted:
break
continue
channels = self._resolve_channels()
cfg = self.snapshot()["config"]
interval_minutes = cfg.get("interval_minutes", 1)
if not channels:
self._log("No channels configured for continuous scraping.", "warn")
else:
self._log(f"Starting iteration for {len(channels)} channel(s).", "info")
buffer = io.StringIO()
try:
with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer):
run_job("scrape_selected", {
"channels": channels,
"account_id": self.account_id,
})
output = buffer.getvalue().strip()
if output:
for line in output.splitlines():
self._log(line)
with self.lock:
self.status["last_iteration_at"] = utc_now_iso()
self.status["last_finished_at"] = utc_now_iso()
self.status["last_error"] = None
self._log("Iteration finished.", "success")
except Exception as exc:
output = buffer.getvalue().strip()
if output:
for line in output.splitlines():
self._log(line)
self._log(f"Iteration failed: {exc}", "error")
with self.lock:
self.status["last_error"] = str(exc)
sleep_seconds = max(5, interval_minutes * 60)
self._log(f"Sleeping for {interval_minutes} minute(s).", "debug")
interrupted = self.stop_event.wait(timeout=sleep_seconds)
if interrupted:
break
with self.lock:
self.status["running"] = False
self._log("Continuous scraping stopped.", "warn")
finally:
# Only mark running=False once the thread has truly exited so the
# status reflects reality (a still-running scrape is not "stopped").
with self.lock:
self.status["running"] = False
self._log("Continuous scraping stopped.", "warn")
# ── ContinuousScrapeOrchestrator ─────────────────────────────────────────
@@ -1012,7 +1234,22 @@ class ContinuousScrapeOrchestrator:
self.start_account(account_id)
def remove_account(self, account_id: str) -> None:
self.stop_account(account_id)
with self.lock:
mgr = self.managers.get(account_id)
if mgr:
mgr.stop()
# Wait for the scrape thread to actually stop before the caller
# deletes the account directory / session files, so rmtree does not
# race with a writer mid-iteration. If the thread is still running
# (e.g. mid-scrape) after the timeout we proceed best-effort and
# log a warning.
if not mgr.join(timeout=REMOVE_ACCOUNT_JOIN_TIMEOUT):
logger.warning(
"Continuous scrape thread for account %r still running after "
"%.1fs; removing account data best-effort",
account_id,
REMOVE_ACCOUNT_JOIN_TIMEOUT,
)
with self.lock:
self.managers.pop(account_id, None)
@@ -1841,6 +2078,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/api/jobs":
return self.send_json(self.app.job_runner.recent_jobs())
if path.startswith("/api/jobs/") and path.endswith("/events"):
# SSE streams pin a worker thread up to 30 minutes, so reject
# cross-site / cross-origin clients (e.g. <img> tags on other
# pages) before opening the stream. Requests without an Origin /
# Sec-Fetch-Site header (curl, same-origin EventSource that omits
# it) are allowed - see _check_same_origin().
if not self._check_same_origin():
return
job_id = path.split("/")[-2]
return self.stream_job_events(job_id)
if path.startswith("/api/jobs/"):
@@ -1936,13 +2180,11 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if sub == ["health"]:
return self.send_json(account_health_summary(account_id, self.app.job_runner))
if sub == ["export"]:
include_secrets = query.get("include_secrets", ["0"])[0].lower() in {"1", "true", "yes"}
payload = {
return self.send_json({
"version": 1,
"account_id": account_id,
"state": export_account_state(load_account(DATA_DIR, account_id), include_secrets=include_secrets),
}
return self.send_json(payload)
"state": export_account_state(load_account(DATA_DIR, account_id)),
})
if sub == ["channels"]:
return self._handle_get_account_channels(account_id)
if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages":
@@ -1978,29 +2220,37 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
def stream_job_events(self, job_id: str) -> None:
if not self.app.job_runner.get_job(job_id):
return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found")
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "keep-alive")
self.end_headers()
runner = self.app.job_runner
stream_id = runner.register_event_stream()
try:
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "keep-alive")
self.end_headers()
last_payload = None
deadline = time.time() + 60 * 30
while time.time() < deadline:
job = self.app.job_runner.get_job(job_id)
if not job:
break
payload = json.dumps(job, ensure_ascii=False)
if payload != last_payload:
try:
self.wfile.write(f"data: {payload}\n\n".encode("utf-8"))
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, OSError):
last_payload = None
deadline = time.time() + 60 * 30
while time.time() < deadline:
if runner.is_stream_revoked(stream_id):
# Capped: this stream is the oldest and must close.
break
last_payload = payload
if job.get("status") in {"completed", "failed"}:
break
time.sleep(1)
job = runner.get_job(job_id)
if not job:
break
payload = json.dumps(job, ensure_ascii=False)
if payload != last_payload:
try:
self.wfile.write(f"data: {payload}\n\n".encode("utf-8"))
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, OSError):
break
last_payload = payload
if job.get("status") in TERMINAL_JOB_STATUSES:
break
time.sleep(1)
finally:
runner.unregister_event_stream(stream_id)
def _handle_get_account_channels(self, account_id: str) -> None:
return self.send_json(list_channels_snapshot(account_id))
@@ -2075,9 +2325,15 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
# ── POST ─────────────────────────────────────────────────────────────
def do_POST(self) -> None:
if not self._check_same_origin():
return
parsed = urllib.parse.urlparse(self.path)
path = parsed.path
body = self.read_json_body()
if body is _JSON_CONTENT_TYPE_REJECTED:
return self.send_error_json(HTTPStatus.UNSUPPORTED_MEDIA_TYPE, "Content-Type must be application/json")
if body is _JSON_BODY_TOO_LARGE:
return self.send_error_json(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, "Request body too large")
if body is None:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "Expected JSON body")
@@ -2097,11 +2353,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
try:
enabled = bool(body.get("enabled"))
interval_minutes = int(body.get("interval_minutes", 1))
channels = [
str(item).strip()
for item in body.get("channels", [])
if str(item).strip()
]
channels, dropped = clean_continuous_channels(body.get("channels", []))
run_all_tracked = bool(body.get("run_all_tracked", True))
if self.app.legacy_account_id:
payload = self.app.continuous_orchestrator.update_for(
@@ -2115,6 +2367,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
payload = {"config": {}, "status": {"running": False, "logs": [], "log_entries": []}}
except Exception as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
if isinstance(payload, dict):
payload["dropped_invalid"] = dropped
return self.send_json(payload)
if path == "/api/auth/credentials":
@@ -2421,33 +2675,62 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_json(payload)
def _handle_post_account_auth_phone_request(self, account_id: str, body: Dict[str, Any]) -> None:
ip = self.client_address[0] if self.client_address else ""
if not _check_auth_throttle(ip, account_id):
return self.send_error_json(
HTTPStatus.TOO_MANY_REQUESTS,
"Too many failed attempts. Try again later.",
)
if not _check_auth_code_cooldown(ip, account_id):
return self.send_error_json(
HTTPStatus.TOO_MANY_REQUESTS,
"Please wait before requesting another code.",
)
phone = str(body.get("phone", "")).strip()
if not phone:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "phone is required")
try:
payload = self.app.auth_manager.request_phone_code(account_id, phone)
except Exception as exc:
_record_auth_failure(ip, account_id)
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
_record_auth_code_request(ip, account_id)
return self.send_json(payload)
def _handle_post_account_auth_phone_submit(self, account_id: str, body: Dict[str, Any]) -> None:
ip = self.client_address[0] if self.client_address else ""
if not _check_auth_throttle(ip, account_id):
return self.send_error_json(
HTTPStatus.TOO_MANY_REQUESTS,
"Too many failed attempts. Try again later.",
)
code = str(body.get("code", "")).strip()
if not code:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "code is required")
try:
payload = self.app.auth_manager.submit_phone_code(account_id, code)
except Exception as exc:
_record_auth_failure(ip, account_id)
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
_record_auth_success(ip, account_id)
return self.send_json(payload)
def _handle_post_account_auth_password(self, account_id: str, body: Dict[str, Any]) -> None:
ip = self.client_address[0] if self.client_address else ""
if not _check_auth_throttle(ip, account_id):
return self.send_error_json(
HTTPStatus.TOO_MANY_REQUESTS,
"Too many failed attempts. Try again later.",
)
password = str(body.get("password", "")).strip()
if not password:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "password is required")
try:
payload = self.app.auth_manager.submit_password(account_id, password)
except Exception as exc:
_record_auth_failure(ip, account_id)
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
_record_auth_success(ip, account_id)
return self.send_json(payload)
def _clean_imported_channels(self, channels: Any) -> Dict[str, Any]:
@@ -2594,11 +2877,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
try:
enabled = bool(body.get("enabled"))
interval_minutes = int(body.get("interval_minutes", 1))
channels = [
str(item).strip()
for item in body.get("channels", [])
if str(item).strip()
]
channels, dropped = clean_continuous_channels(body.get("channels", []))
run_all_tracked = bool(body.get("run_all_tracked", True))
payload = self.app.continuous_orchestrator.update_for(
account_id=account_id,
@@ -2609,11 +2888,14 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
except Exception as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
payload["dropped_invalid"] = dropped
return self.send_json(payload)
# ── DELETE ───────────────────────────────────────────────────────────
def do_DELETE(self) -> None:
if not self._check_same_origin():
return
parsed = urllib.parse.urlparse(self.path)
path = parsed.path
@@ -2664,16 +2946,49 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
# ── Helpers ─────────────────────────────────────────────────────────
def read_json_body(self) -> Optional[Dict[str, Any]]:
length = int(self.headers.get("Content-Length", "0"))
def read_json_body(self) -> Any:
try:
length = int(self.headers.get("Content-Length", "0"))
except (TypeError, ValueError):
# Unparseable Content-Length - treat as no body rather than 500.
length = 0
if length <= 0:
return {}
if length > MAX_JSON_BODY_BYTES:
return _JSON_BODY_TOO_LARGE
content_type = self.headers.get("Content-Type", "")
media_type = content_type.split(";", 1)[0].strip().lower()
if media_type != "application/json":
return _JSON_CONTENT_TYPE_REJECTED
raw = self.rfile.read(length)
if len(raw) > MAX_JSON_BODY_BYTES:
return _JSON_BODY_TOO_LARGE
try:
return json.loads(raw.decode("utf-8"))
except json.JSONDecodeError:
except (json.JSONDecodeError, UnicodeDecodeError):
return None
def _check_same_origin(self) -> bool:
"""Reject state-mutating requests that are clearly cross-origin.
Returns True when the request may proceed. Requests without an
Origin or Sec-Fetch-Site header are allowed (curl, same-origin
browsers that omit the header). When present, the headers must not
indicate a cross-origin request.
"""
host = self.headers.get("Host", "")
origin = self.headers.get("Origin")
if origin:
origin_host = urllib.parse.urlparse(origin).netloc
if not origin_host or (host and origin_host != host):
self.send_error_json(HTTPStatus.FORBIDDEN, "Cross-origin request rejected")
return False
sec_fetch_site = self.headers.get("Sec-Fetch-Site", "").strip().lower()
if sec_fetch_site and sec_fetch_site not in {"same-origin", "same-site", "none"}:
self.send_error_json(HTTPStatus.FORBIDDEN, "Cross-origin request rejected")
return False
return True
def serve_static(self, relative_path: str, head_only: bool = False) -> None:
file_path = (WEBUI_DIR / relative_path).resolve()
try: