forust/telegram-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Activity

remove REVIEW.md from repo

Browse Source
This commit is contained in:
mr-forust
2026-09-10 11:50:12 +02:00
parent 4c12e8bb0c
commit b92f8d8914
15 changed files with 397 additions and 639 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui_server.py
+119 -135
View File
@@ -16,31 +16,31 @@ import traceback
import urllib.parse
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from datetime import UTC, datetime
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from typing import Any
import qrcode
import qrcode.image.svg
from telethon import TelegramClient
from telethon.errors import SessionPasswordNeededError
from app_state import (
StateStore,
account_exists,
account_data_dir,
account_exists,
account_session_path,
get_account_store,
get_global_store,
load_account,
list_accounts,
load_account,
)
from health import health_payload
from scraper_jobs import ScraperJobService
from telethon import TelegramClient
from telethon.errors import SessionPasswordNeededError
from telegram_scraper_with_forwarding import _ensure_session_wal
BASE_DIR = Path(__file__).resolve().parent
DATA_DIR = BASE_DIR / "data"
WEBUI_DIR = BASE_DIR / "webui"
@@ -88,7 +88,7 @@ AUTH_LOCKOUT_SECONDS = 60
AUTH_CODE_REQUEST_COOLDOWN_SECONDS = 30
# Cap the in-memory auth tracking dict; beyond this we sweep expired entries.
AUTH_ATTEMPTS_MAX_ENTRIES = 10_000
_auth_attempts: Dict[Tuple[str, str], Dict[str, Any]] = {}
_auth_attempts: dict[tuple[str, str], dict[str, Any]] = {}
_auth_attempts_lock = threading.Lock()
# ── M-3: boolean coercion helper ─────────────────────────────────────────
@@ -116,7 +116,7 @@ def parse_bool(value: Any, default: bool = False) -> bool:
# ── L-8: trusted-host check ──────────────────────────────────────────────
import ipaddress # noqa: E402
import ipaddress
# Comma-separated extra trusted hostnames for browser state-mutating
# requests when the panel is served through a proxy domain.
@@ -211,9 +211,7 @@ def _check_auth_code_cooldown(ip: str, account_id: str) -> bool:
if not entry:
return True
cooldown_until = entry.get("cooldown_until")
if cooldown_until and now < cooldown_until:
return False
return True
return not (cooldown_until and now < cooldown_until)
def _record_auth_failure(ip: str, account_id: str) -> None:
@@ -248,18 +246,18 @@ def _record_auth_code_request(ip: str, account_id: str) -> None:
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
return datetime.now(UTC).isoformat()
def load_state() -> Dict[str, Any]:
def load_state() -> dict[str, Any]:
return STATE_STORE.load()
def save_state(state: Dict[str, Any]) -> None:
def save_state(state: dict[str, Any]) -> None:
STATE_STORE.save(state)
def guess_media_kind(media_path: Optional[str], media_type: Optional[str]) -> Optional[str]:
def guess_media_kind(media_path: str | None, media_type: str | None) -> str | None:
if not media_path:
return None
suffix = Path(media_path).suffix.lower()
@@ -274,7 +272,7 @@ def guess_media_kind(media_path: Optional[str], media_type: Optional[str]) -> Op
return "file"
def normalize_media_url(media_path: Optional[str]) -> Optional[str]:
def normalize_media_url(media_path: str | None) -> str | None:
if not media_path:
return None
path = Path(media_path)
@@ -304,7 +302,7 @@ def normalize_channel_id(value: Any) -> str:
return channel_id
def clean_continuous_channels(channels: Any) -> Tuple[List[str], List[str]]:
def clean_continuous_channels(channels: Any) -> tuple[list[str], list[str]]:
"""Validate/normalize a continuous-scrape channel list at ingest.
Returns ``(cleaned, dropped)`` where ``cleaned`` holds the normalized,
@@ -313,8 +311,8 @@ def clean_continuous_channels(channels: Any) -> Tuple[List[str], List[str]]:
dropped entries must NOT be persisted so they cannot become a path
traversal vector for any future direct read.
"""
cleaned: List[str] = []
dropped: List[str] = []
cleaned: list[str] = []
dropped: list[str] = []
if not isinstance(channels, list):
return cleaned, dropped
for item in channels:
@@ -326,7 +324,7 @@ def clean_continuous_channels(channels: Any) -> Tuple[List[str], List[str]]:
return cleaned, dropped
def export_account_state(state: Dict[str, Any]) -> Dict[str, Any]:
def export_account_state(state: dict[str, Any]) -> dict[str, Any]:
"""Return a sanitized copy of an account state for export.
Credentials (api_hash, api_id) are always redacted - they are never
@@ -342,15 +340,15 @@ def export_account_state(state: Dict[str, Any]) -> Dict[str, Any]:
return exported
def channel_db_path(account_id: Optional[str], channel_id: str) -> Path:
def channel_db_path(account_id: str | None, channel_id: str) -> Path:
if account_id:
return account_data_dir(DATA_DIR, account_id) / channel_id / f"{channel_id}.db"
return DATA_DIR / channel_id / f"{channel_id}.db"
def database_summary(account_id: Optional[str], channel_id: str) -> Dict[str, Any]:
def database_summary(account_id: str | None, channel_id: str) -> dict[str, Any]:
db_path = channel_db_path(account_id, channel_id)
summary: Dict[str, Any] = {
summary: dict[str, Any] = {
"message_count": 0,
"last_date": None,
"first_date": None,
@@ -398,16 +396,16 @@ def database_summary(account_id: Optional[str], channel_id: str) -> Dict[str, An
conn.close()
def channel_display_name(state: Dict[str, Any], channel_id: str) -> str:
def channel_display_name(state: dict[str, Any], channel_id: str) -> str:
return state.get("channel_names", {}).get(channel_id) or channel_id
def list_channels_snapshot(account_id: Optional[str] = None) -> List[Dict[str, Any]]:
def list_channels_snapshot(account_id: str | None = None) -> list[dict[str, Any]]:
if account_id:
state = load_account(DATA_DIR, account_id)
else:
state = load_state()
channels: List[Dict[str, Any]] = []
channels: list[dict[str, Any]] = []
for channel_id, last_message_id in state.get("channels", {}).items():
summary = database_summary(account_id, channel_id)
channels.append(
@@ -425,7 +423,7 @@ def list_channels_snapshot(account_id: Optional[str] = None) -> List[Dict[str, A
return channels
def account_state_for_legacy() -> Optional[Dict[str, Any]]:
def account_state_for_legacy() -> dict[str, Any] | None:
legacy = load_state()
if legacy.get("api_id") and legacy.get("api_hash"):
return legacy
@@ -433,12 +431,12 @@ def account_state_for_legacy() -> Optional[Dict[str, Any]]:
def load_messages(
account_id: Optional[str],
account_id: str | None,
channel_id: str,
limit: int = 120,
before_message_id: Optional[int] = None,
search: Optional[str] = None,
) -> List[Dict[str, Any]]:
before_message_id: int | None = None,
search: str | None = None,
) -> list[dict[str, Any]]:
db_path = channel_db_path(account_id, channel_id)
if not db_path.exists():
return []
@@ -450,8 +448,8 @@ def load_messages(
"media_type, media_path, reply_to, post_author, views, forwards, reactions "
"FROM messages "
)
params: List[Any] = []
where: List[str] = []
params: list[Any] = []
where: list[str] = []
if before_message_id is not None:
where.append("message_id < ?")
params.append(before_message_id)
@@ -466,7 +464,7 @@ def load_messages(
finally:
conn.close()
messages: List[Dict[str, Any]] = []
messages: list[dict[str, Any]] = []
for row in reversed(rows):
sender_name = " ".join(
part for part in [row["first_name"], row["last_name"]] if part
@@ -505,7 +503,7 @@ def load_messages(
f"FROM messages WHERE message_id IN ({placeholders})",
reply_ids,
).fetchall()
reply_map: Dict[int, Dict[str, str]] = {}
reply_map: dict[int, dict[str, str]] = {}
for r in rows2:
r_sender = " ".join(
part for part in [r["first_name"], r["last_name"]] if part
@@ -533,16 +531,16 @@ class Job:
job_id: str
job_type: str
title: str
payload: Dict[str, Any]
payload: dict[str, Any]
status: str = "queued"
created_at: str = field(default_factory=utc_now_iso)
started_at: Optional[str] = None
finished_at: Optional[str] = None
started_at: str | None = None
finished_at: str | None = None
logs: str = ""
error: Optional[str] = None
account_id: Optional[str] = None
error: str | None = None
account_id: str | None = None
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
return {
"job_id": self.job_id,
"job_type": self.job_type,
@@ -562,14 +560,14 @@ class Job:
class JobRunner:
def __init__(self) -> None:
self.jobs: Dict[str, Job] = {}
self.job_order: List[str] = []
self.queue: "queue.Queue[Job]" = queue.Queue()
self.jobs: dict[str, Job] = {}
self.job_order: list[str] = []
self.queue: queue.Queue[Job] = queue.Queue()
self.lock = threading.Lock()
self._shutdown_flag = False
# Active SSE event streams: stream_id -> started_at, plus any stream
# ids that have been revoked (told to close) because the cap was hit.
self._stream_started: Dict[str, float] = {}
self._stream_started: dict[str, float] = {}
self._stream_revoked: set = set()
self.worker = threading.Thread(target=self._run, daemon=True)
self.worker.start()
@@ -594,7 +592,7 @@ class JobRunner:
with self.lock:
return stream_id in self._stream_revoked
def create_job(self, job_type: str, title: str, payload: Dict[str, Any]) -> Job:
def create_job(self, job_type: str, title: str, payload: dict[str, Any]) -> Job:
if self._shutdown_flag:
raise RuntimeError("Server is shutting down, cannot create new jobs")
account_id = payload.get("account_id")
@@ -626,14 +624,14 @@ class JobRunner:
self.queue.put(job)
return job
def recent_jobs(self, account_id: Optional[str] = None) -> List[Dict[str, Any]]:
def recent_jobs(self, account_id: str | None = None) -> list[dict[str, Any]]:
with self.lock:
result = [self.jobs[job_id].to_dict() for job_id in self.job_order]
if account_id is not None:
result = [j for j in result if j.get("account_id") == account_id]
return result
def active_jobs_by_account(self) -> Dict[str, Dict[str, Any]]:
def active_jobs_by_account(self) -> dict[str, dict[str, Any]]:
with self.lock:
active = {}
for job_id in self.job_order:
@@ -642,7 +640,7 @@ class JobRunner:
active[job.account_id] = job.to_dict()
return active
def get_job(self, job_id: str) -> Optional[Dict[str, Any]]:
def get_job(self, job_id: str) -> dict[str, Any] | None:
with self.lock:
job = self.jobs.get(job_id)
return job.to_dict() if job else None
@@ -731,8 +729,8 @@ class TelegramAuthManager:
self.loop = asyncio.new_event_loop()
self.thread = threading.Thread(target=self._run_loop, daemon=True)
self.thread.start()
self.clients: Dict[str, TelegramClient] = {}
self.auth_data: Dict[str, Dict[str, Any]] = {}
self.clients: dict[str, TelegramClient] = {}
self.auth_data: dict[str, dict[str, Any]] = {}
def _run_loop(self) -> None:
asyncio.set_event_loop(self.loop)
@@ -742,7 +740,7 @@ class TelegramAuthManager:
future = asyncio.run_coroutine_threadsafe(coro, self.loop)
return future.result()
def _get_auth_data(self, account_id: str) -> Dict[str, Any]:
def _get_auth_data(self, account_id: str) -> dict[str, Any]:
with self.lock:
if account_id not in self.auth_data:
self.auth_data[account_id] = {
@@ -808,7 +806,7 @@ class TelegramAuthManager:
data["user_id"] = me.id
return client
def auth_state(self, account_id: str) -> Dict[str, Any]:
def auth_state(self, account_id: str) -> dict[str, Any]:
with self.lock:
data = self._get_auth_data(account_id)
snapshot = dict(data)
@@ -833,14 +831,14 @@ class TelegramAuthManager:
snapshot["auth_status"] = auth_status_for(account_id)
return snapshot
def all_auth_states(self) -> Dict[str, Dict[str, Any]]:
result: Dict[str, Dict[str, Any]] = {}
def all_auth_states(self) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = {}
for account_id in list_accounts(DATA_DIR):
result[account_id] = self.auth_state(account_id)
return result
def save_credentials(self, account_id: str, api_id: int, api_hash: str) -> Dict[str, Any]:
def mutate(state: Dict[str, Any]) -> None:
def save_credentials(self, account_id: str, api_id: int, api_hash: str) -> dict[str, Any]:
def mutate(state: dict[str, Any]) -> None:
state["api_id"] = int(api_id)
state["api_hash"] = api_hash.strip()
@@ -854,7 +852,7 @@ class TelegramAuthManager:
)
return self.auth_state(account_id)
async def _start_qr_login(self, account_id: str) -> Dict[str, Any]:
async def _start_qr_login(self, account_id: str) -> dict[str, Any]:
client = await self._get_client(account_id)
if await client.is_user_authorized():
self._set_state(
@@ -923,10 +921,10 @@ class TelegramAuthManager:
qr_created_at=None,
)
def start_qr_login(self, account_id: str) -> Dict[str, Any]:
def start_qr_login(self, account_id: str) -> dict[str, Any]:
return self._run(self._start_qr_login(account_id))
async def _request_phone_code(self, account_id: str, phone: str) -> Dict[str, Any]:
async def _request_phone_code(self, account_id: str, phone: str) -> dict[str, Any]:
client = await self._get_client(account_id)
if await client.is_user_authorized():
self._set_state(
@@ -947,10 +945,10 @@ class TelegramAuthManager:
)
return self.auth_state(account_id)
def request_phone_code(self, account_id: str, phone: str) -> Dict[str, Any]:
def request_phone_code(self, account_id: str, phone: str) -> dict[str, Any]:
return self._run(self._request_phone_code(account_id, phone))
async def _submit_phone_code(self, account_id: str, code: str) -> Dict[str, Any]:
async def _submit_phone_code(self, account_id: str, code: str) -> dict[str, Any]:
client = await self._get_client(account_id)
with self.lock:
data = self._get_auth_data(account_id)
@@ -982,10 +980,10 @@ class TelegramAuthManager:
)
return self.auth_state(account_id)
def submit_phone_code(self, account_id: str, code: str) -> Dict[str, Any]:
def submit_phone_code(self, account_id: str, code: str) -> dict[str, Any]:
return self._run(self._submit_phone_code(account_id, code))
async def _submit_password(self, account_id: str, password: str) -> Dict[str, Any]:
async def _submit_password(self, account_id: str, password: str) -> dict[str, Any]:
client = await self._get_client(account_id)
await client.sign_in(password=password)
self._set_state(
@@ -999,7 +997,7 @@ class TelegramAuthManager:
)
return self.auth_state(account_id)
def submit_password(self, account_id: str, password: str) -> Dict[str, Any]:
def submit_password(self, account_id: str, password: str) -> dict[str, Any]:
return self._run(self._submit_password(account_id, password))
def delete_account(self, account_id: str) -> None:
@@ -1049,14 +1047,14 @@ class PerAccountContinuousScrapeManager:
def __init__(self, account_id: str) -> None:
self.account_id = account_id
self.lock = threading.RLock()
self.thread: Optional[threading.Thread] = None
self.thread: threading.Thread | None = None
self.stop_event = threading.Event()
# F-4: set on remove_account when its thread could not be joined in
# time; the manager is kept as a tombstone so it is reused (after the
# drain) instead of spawning a duplicate worker on account re-add.
self._removing = False
self.config: Dict[str, Any] = self._load_config()
self.status: Dict[str, Any] = {
self.config: dict[str, Any] = self._load_config()
self.status: dict[str, Any] = {
"running": False,
"last_started_at": None,
"last_finished_at": None,
@@ -1066,7 +1064,7 @@ class PerAccountContinuousScrapeManager:
"log_entries": [],
}
def _load_config(self) -> Dict[str, Any]:
def _load_config(self) -> dict[str, Any]:
acc_state = load_account(DATA_DIR, self.account_id)
cfg = dict(acc_state.get("continuous_scraping", {
"enabled": True,
@@ -1085,7 +1083,7 @@ class PerAccountContinuousScrapeManager:
def _save_config(self) -> None:
store = get_account_store(DATA_DIR, self.account_id)
def mutate(state: Dict[str, Any]) -> None:
def mutate(state: dict[str, Any]) -> None:
state["continuous_scraping"] = {
"enabled": bool(self.config.get("enabled", True)),
"interval_minutes": max(1, int(self.config.get("interval_minutes", 1) or 1)),
@@ -1140,7 +1138,7 @@ class PerAccountContinuousScrapeManager:
if len(log_entries) > 300:
del log_entries[:-300]
def snapshot(self) -> Dict[str, Any]:
def snapshot(self) -> dict[str, Any]:
with self.lock:
return {
"config": dict(self.config),
@@ -1155,9 +1153,9 @@ class PerAccountContinuousScrapeManager:
self,
enabled: bool,
interval_minutes: int,
channels: List[str],
channels: list[str],
run_all_tracked: bool,
) -> Dict[str, Any]:
) -> dict[str, Any]:
interval_minutes = max(1, int(interval_minutes))
with self.lock:
self.config = {
@@ -1242,7 +1240,7 @@ class PerAccountContinuousScrapeManager:
thread.join(timeout=timeout)
return not thread.is_alive()
def _resolve_channels(self) -> List[str]:
def _resolve_channels(self) -> list[str]:
acc_state = load_account(DATA_DIR, self.account_id)
with self.lock:
run_all_tracked = self.config.get("run_all_tracked", True)
@@ -1344,7 +1342,7 @@ class PerAccountContinuousScrapeManager:
class ContinuousScrapeOrchestrator:
def __init__(self) -> None:
self.lock = threading.Lock()
self.managers: Dict[str, PerAccountContinuousScrapeManager] = {}
self.managers: dict[str, PerAccountContinuousScrapeManager] = {}
def _get_or_create(self, account_id: str) -> PerAccountContinuousScrapeManager:
with self.lock:
@@ -1371,12 +1369,12 @@ class ContinuousScrapeOrchestrator:
if mgr:
mgr.stop()
def snapshot_for(self, account_id: str) -> Dict[str, Any]:
def snapshot_for(self, account_id: str) -> dict[str, Any]:
mgr = self._get_or_create(account_id)
return mgr.snapshot()
def snapshot(self) -> Dict[str, Dict[str, Any]]:
result: Dict[str, Dict[str, Any]] = {}
def snapshot(self) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = {}
with self.lock:
for account_id, mgr in self.managers.items():
result[account_id] = dict(mgr.snapshot())
@@ -1396,9 +1394,9 @@ class ContinuousScrapeOrchestrator:
account_id: str,
enabled: bool,
interval_minutes: int,
channels: List[str],
channels: list[str],
run_all_tracked: bool,
) -> Dict[str, Any]:
) -> dict[str, Any]:
mgr = self._get_or_create(account_id)
result = mgr.update(enabled, interval_minutes, channels, run_all_tracked)
# After explicit update, sync state to disk is already done by mgr.update
@@ -1456,18 +1454,18 @@ def import_scraper_class():
return OptimizedTelegramScraper
def run_job(job_type: str, payload: Dict[str, Any]) -> None:
def run_job(job_type: str, payload: dict[str, Any]) -> None:
SCRAPER_JOBS.run(job_type, payload)
def auth_status_for(account_id: Optional[str] = None) -> Dict[str, Any]:
def auth_status_for(account_id: str | None = None) -> dict[str, Any]:
if account_id:
acc_state = load_account(DATA_DIR, account_id)
session_file = account_session_path(SESSION_DIR, account_id)
else:
acc_state = load_state()
session_file = str(SESSION_DIR / "session.session")
status: Dict[str, Any] = {
status: dict[str, Any] = {
"has_api_credentials": bool(acc_state.get("api_id") and acc_state.get("api_hash")),
"telethon_available": False,
"session_ready": False,
@@ -1494,7 +1492,7 @@ def auth_status_for(account_id: Optional[str] = None) -> Dict[str, Any]:
return status
def account_health_summary(account_id: str, job_runner: Optional[JobRunner] = None) -> Dict[str, Any]:
def account_health_summary(account_id: str, job_runner: JobRunner | None = None) -> dict[str, Any]:
acc_state = load_account(DATA_DIR, account_id)
acc_dir = account_data_dir(DATA_DIR, account_id)
session_file = Path(account_session_path(SESSION_DIR, account_id))
@@ -1515,11 +1513,11 @@ def account_health_summary(account_id: str, job_runner: Optional[JobRunner] = No
}
def auth_status() -> Dict[str, Any]:
def auth_status() -> dict[str, Any]:
return auth_status_for(account_id=None)
def dashboard_payload(job_runner: JobRunner, account_id: Optional[str] = None) -> Dict[str, Any]:
def dashboard_payload(job_runner: JobRunner, account_id: str | None = None) -> dict[str, Any]:
if account_id:
acc_state = load_account(DATA_DIR, account_id)
state = {
@@ -1547,7 +1545,7 @@ def dashboard_payload(job_runner: JobRunner, account_id: Optional[str] = None) -
}
def openapi_payload() -> Dict[str, Any]:
def openapi_payload() -> dict[str, Any]:
json_response = {
"200": {
"description": "JSON response",
@@ -1562,7 +1560,7 @@ def openapi_payload() -> Dict[str, Any]:
}
error_response = {"400": {"description": "Bad request"}}
def json_body(properties: Dict[str, Any], required: Optional[List[str]] = None):
def json_body(properties: dict[str, Any], required: list[str] | None = None):
return {
"required": True,
"content": {
@@ -2335,7 +2333,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.NOT_FOUND, "Not found")
def _legacy_auth_snapshot(self) -> Dict[str, Any]:
def _legacy_auth_snapshot(self) -> dict[str, Any]:
state = load_state()
return {
"phase": "unknown",
@@ -2369,7 +2367,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
})
return self.send_json({"accounts": accounts})
def _handle_get_account(self, path: str, query: Dict[str, List[str]]) -> None:
def _handle_get_account(self, path: str, query: dict[str, list[str]]) -> None:
rest = path[len("/api/accounts/"):]
parts = rest.split("/")
account_id = urllib.parse.unquote(parts[0])
@@ -2446,7 +2444,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
payload = json.dumps(job, ensure_ascii=False)
if payload != last_payload:
try:
self.wfile.write(f"data: {payload}\n\n".encode("utf-8"))
self.wfile.write(f"data: {payload}\n\n".encode())
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, OSError):
break
@@ -2461,7 +2459,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_json(list_channels_snapshot(account_id))
def _handle_get_account_channel_messages(
self, account_id: str, channel_id: str, query: Dict[str, List[str]]
self, account_id: str, channel_id: str, query: dict[str, list[str]]
) -> None:
try:
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
@@ -2511,21 +2509,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
relative = path.removeprefix("/media/")
return self.serve_media(relative, head_only=True)
if (
path
in {
"/api/dashboard",
"/api/auth",
"/api/continuous",
"/api/jobs",
"/api/channels",
"/api/accounts",
"/health",
"/health/continuous",
"/openapi.json",
}
or path.startswith("/api/jobs/")
or (path.startswith("/api/channels/") and path.endswith("/messages"))
or path.startswith("/api/accounts/")
path in {"/api/dashboard", "/api/auth", "/api/continuous", "/api/jobs", "/api/channels", "/api/accounts", "/health", "/health/continuous", "/openapi.json"} or path.startswith(("/api/jobs/", "/api/accounts/")) or path.startswith("/api/channels/") and path.endswith("/messages")
):
# M-4: for the job-events route, mirror GET's behavior and verify
# the job actually exists before responding with 200.
@@ -2562,7 +2546,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
# No legacy account: persist to the (global) legacy store directly.
# The ScraperJobService set_scrape_media job is a passthrough, so
# enqueueing it alone would silently drop the setting.
def _media_mutate(global_state: Dict[str, Any]) -> None:
def _media_mutate(global_state: dict[str, Any]) -> None:
global_state["scrape_media"] = value
STATE_STORE.update(_media_mutate)
return self.send_json({"ok": True, "scrape_media": value})
@@ -2655,7 +2639,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if self.app.legacy_account_id:
return self._handle_account_channel_add(self.app.legacy_account_id, channel_id, body.get("name"))
# L-6: use the atomic mutator to avoid lost-update races.
def _add_mutate(state: Dict[str, Any]) -> None:
def _add_mutate(state: dict[str, Any]) -> None:
if channel_id not in state.get("channels", {}):
state.setdefault("channels", {})[channel_id] = 0
if body.get("name"):
@@ -2672,7 +2656,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self._handle_account_channel_remove(self.app.legacy_account_id, channel_id)
# L-6: use the atomic mutator to avoid lost-update races.
existed = [False]
def _remove_mutate(state: Dict[str, Any]) -> None:
def _remove_mutate(state: dict[str, Any]) -> None:
chans = state.get("channels", {})
if channel_id in chans:
existed[0] = True
@@ -2782,7 +2766,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.NOT_FOUND, "Not found")
def _handle_post_accounts_create(self, body: Dict[str, Any]) -> None:
def _handle_post_accounts_create(self, body: dict[str, Any]) -> None:
account_id = str(body.get("account_id", "")).strip()
if not account_id:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id is required")
@@ -2797,7 +2781,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
api_id = body.get("api_id")
api_hash = str(body.get("api_hash", "")).strip()
def mutate_global(state: Dict[str, Any]) -> None:
def mutate_global(state: dict[str, Any]) -> None:
accounts = state.setdefault("accounts", [])
if account_id not in accounts:
accounts.append(account_id)
@@ -2824,7 +2808,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.app.continuous_orchestrator.add_account(account_id, auto_start=False)
return self.send_json({"ok": True, "account_id": account_id})
def _handle_post_accounts_import(self, body: Dict[str, Any]) -> None:
def _handle_post_accounts_import(self, body: dict[str, Any]) -> None:
account_id = str(body.get("account_id") or body.get("id") or "").strip()
state = body.get("state")
if not account_id:
@@ -2863,7 +2847,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
},
}
def mutate_global(global_state: Dict[str, Any]) -> None:
def mutate_global(global_state: dict[str, Any]) -> None:
accounts = global_state.setdefault("accounts", [])
if account_id not in accounts:
accounts.append(account_id)
@@ -2874,7 +2858,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.app.continuous_orchestrator.refresh_account(account_id)
return self.send_json({"ok": True, "account_id": account_id})
def _handle_post_account(self, path: str, body: Dict[str, Any]) -> None:
def _handle_post_account(self, path: str, body: dict[str, Any]) -> None:
rest = path[len("/api/accounts/"):]
parts = rest.split("/")
account_id = urllib.parse.unquote(parts[0])
@@ -2922,7 +2906,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return False
return True
def _handle_post_account_auth_credentials(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_post_account_auth_credentials(self, account_id: str, body: dict[str, Any]) -> None:
api_id = body.get("api_id")
api_hash = str(body.get("api_hash", "")).strip()
if not api_id or not api_hash:
@@ -2940,7 +2924,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_bad_request_from_exc(exc, fallback="Failed to start QR login")
return self.send_json(payload)
def _handle_post_account_auth_phone_request(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_post_account_auth_phone_request(self, account_id: str, body: dict[str, Any]) -> None:
ip = self.client_address[0] if self.client_address else ""
if not _check_auth_throttle(ip, account_id):
return self.send_error_json(
@@ -2963,7 +2947,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
_record_auth_code_request(ip, account_id)
return self.send_json(payload)
def _handle_post_account_auth_phone_submit(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_post_account_auth_phone_submit(self, account_id: str, body: dict[str, Any]) -> None:
ip = self.client_address[0] if self.client_address else ""
if not _check_auth_throttle(ip, account_id):
return self.send_error_json(
@@ -2981,7 +2965,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
_record_auth_success(ip, account_id)
return self.send_json(payload)
def _handle_post_account_auth_password(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_post_account_auth_password(self, account_id: str, body: dict[str, Any]) -> None:
ip = self.client_address[0] if self.client_address else ""
if not _check_auth_throttle(ip, account_id):
return self.send_error_json(
@@ -2999,10 +2983,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
_record_auth_success(ip, account_id)
return self.send_json(payload)
def _clean_imported_channels(self, channels: Any) -> Dict[str, Any]:
def _clean_imported_channels(self, channels: Any) -> dict[str, Any]:
if not isinstance(channels, dict):
return {}
cleaned: Dict[str, Any] = {}
cleaned: dict[str, Any] = {}
for raw_channel_id, last_message_id in channels.items():
try:
channel_id = normalize_channel_id(raw_channel_id)
@@ -3011,10 +2995,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
cleaned[channel_id] = last_message_id
return cleaned
def _clean_imported_channel_names(self, channel_names: Any) -> Dict[str, str]:
def _clean_imported_channel_names(self, channel_names: Any) -> dict[str, str]:
if not isinstance(channel_names, dict):
return {}
cleaned: Dict[str, str] = {}
cleaned: dict[str, str] = {}
for raw_channel_id, name in channel_names.items():
try:
channel_id = normalize_channel_id(raw_channel_id)
@@ -3023,13 +3007,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
cleaned[channel_id] = str(name).strip()
return cleaned
def _handle_account_channel_add(self, account_id: str, channel_id: Any, name: Optional[str]) -> None:
def _handle_account_channel_add(self, account_id: str, channel_id: Any, name: str | None) -> None:
try:
channel_id = normalize_channel_id(channel_id)
except ValueError as exc:
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
store = get_account_store(DATA_DIR, account_id)
def mutate(state: Dict[str, Any]) -> None:
def mutate(state: dict[str, Any]) -> None:
if channel_id not in state.setdefault("channels", {}):
state["channels"][channel_id] = 0
if name:
@@ -3045,7 +3029,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
store = get_account_store(DATA_DIR, account_id)
existed = [False]
def mutate(state: Dict[str, Any]) -> None:
def mutate(state: dict[str, Any]) -> None:
chans = state.get("channels", {})
if channel_id in chans:
existed[0] = True
@@ -3055,7 +3039,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.app.continuous_orchestrator.refresh_account(account_id)
return self.send_json({"ok": existed[0], "channel_id": channel_id})
def _handle_account_job_scrape(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_account_job_scrape(self, account_id: str, body: dict[str, Any]) -> None:
channel_id = body.get("channel_id")
if channel_id:
try:
@@ -3083,7 +3067,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
def _handle_account_job_export_channel(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_account_job_export_channel(self, account_id: str, body: dict[str, Any]) -> None:
try:
channel_id = normalize_channel_id(body.get("channel_id"))
except ValueError as exc:
@@ -3095,7 +3079,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
def _handle_account_job_rescrape_media(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_account_job_rescrape_media(self, account_id: str, body: dict[str, Any]) -> None:
try:
channel_id = normalize_channel_id(body.get("channel_id"))
except ValueError as exc:
@@ -3107,7 +3091,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
def _handle_account_job_fix_media(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_account_job_fix_media(self, account_id: str, body: dict[str, Any]) -> None:
try:
channel_id = normalize_channel_id(body.get("channel_id"))
except ValueError as exc:
@@ -3129,7 +3113,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
def _handle_account_settings_media(self, account_id: str, value: bool) -> None:
store = get_account_store(DATA_DIR, account_id)
def mutate(state: Dict[str, Any]) -> None:
def mutate(state: dict[str, Any]) -> None:
state["scrape_media"] = value
store.update(mutate)
job = self.app.job_runner.create_job(
@@ -3139,7 +3123,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
def _handle_account_continuous(self, account_id: str, body: Dict[str, Any]) -> None:
def _handle_account_continuous(self, account_id: str, body: dict[str, Any]) -> None:
try:
enabled = parse_bool(body.get("enabled"))
interval_minutes = int(body.get("interval_minutes", 1))
@@ -3199,7 +3183,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.app.continuous_orchestrator.remove_account(account_id)
self.app.auth_manager.delete_account(account_id)
def mutate_global(state: Dict[str, Any]) -> None:
def mutate_global(state: dict[str, Any]) -> None:
accounts = state.setdefault("accounts", [])
if account_id in accounts:
accounts.remove(account_id)
@@ -3456,7 +3440,7 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
self.job_runner = JobRunner()
self.auth_manager = TelegramAuthManager()
self.continuous_orchestrator = ContinuousScrapeOrchestrator()
self.legacy_account_id: Optional[str] = None
self.legacy_account_id: str | None = None
self._detect_legacy_account()
if START_CONTINUOUS: