From db3641ddf91553005c8154b027fb887961bf9f06 Mon Sep 17 00:00:00 2001 From: mr-forust Date: Sat, 25 Apr 2026 02:34:35 +0200 Subject: [PATCH] Add web auth and continuous scraping logs --- .github/FUNDING.yml | 1 - .gitignore | 5 +- compose.yaml | 4 +- webui/app.js | 122 ++++++++++++- webui/index.html | 70 ++++++++ webui/style.css | 43 +++++ webui_server.py | 405 +++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 640 insertions(+), 10 deletions(-) delete mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 45b4275..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -ko_fi: unnohwn diff --git a/.gitignore b/.gitignore index 25dd951..ed0520f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,8 @@ data/ .python-version uv.lock .uv-cache/ - +session/ +session.session # Byte-compiled / optimized / DLL files __pycache__/ @@ -224,4 +225,4 @@ marimo/_lsp/ __marimo__/ # Streamlit -.streamlit/secrets.toml \ No newline at end of file +.streamlit/secrets.toml diff --git a/compose.yaml b/compose.yaml index 93be0ab..50f6617 100644 --- a/compose.yaml +++ b/compose.yaml @@ -12,6 +12,4 @@ services: - "8080:8080" volumes: - ./data:/app/data - - session:/app/session -volumes: - session: + - ./session:/app/session diff --git a/webui/app.js b/webui/app.js index 7f3d569..b52d5e9 100644 --- a/webui/app.js +++ b/webui/app.js @@ -1,5 +1,7 @@ const state = { dashboard: null, + auth: null, + continuous: null, }; async function api(path, options = {}) { @@ -29,6 +31,30 @@ function renderAuth(auth) { document.getElementById("auth-details").textContent = auth.details || ""; } +function toggleHidden(id, hidden) { + document.getElementById(id).classList.toggle("hidden", hidden); +} + +function renderAuthPanel(authData) { + state.auth = authData; + renderAuth(authData.auth_status); + + const apiId = authData.saved_credentials?.api_id ?? ""; + document.getElementById("api-id-input").value = apiId || ""; + document.getElementById("api-hash-input").placeholder = authData.saved_credentials?.api_hash_present + ? "API Hash сохранён" + : "API Hash"; + + const showQr = Boolean(authData.qr_image); + toggleHidden("qr-wrap", !showQr); + if (showQr) { + document.getElementById("qr-image").src = authData.qr_image; + } + + toggleHidden("code-form", authData.phase !== "code_required"); + toggleHidden("password-form", authData.phase !== "password_required"); +} + function renderSummary(data) { document.getElementById("channel-count").textContent = String(data.state.channel_count); document.getElementById("forwarding-count").textContent = String(data.state.forwarding_rules.length); @@ -102,13 +128,32 @@ function renderJobs(jobs) { }); } +function renderContinuous(data) { + state.continuous = data; + const config = data.config || {}; + const status = data.status || {}; + + document.getElementById("continuous-enabled").checked = Boolean(config.enabled); + document.getElementById("continuous-interval").value = config.interval_minutes || 1; + document.getElementById("continuous-all").checked = Boolean(config.run_all_tracked); + document.getElementById("continuous-channels").value = (config.channels || []).join(", "); + + document.getElementById("continuous-status").textContent = status.running ? "running" : "stopped"; + document.getElementById("continuous-last-iteration").textContent = status.last_iteration_at || "—"; + document.getElementById("continuous-last-error").textContent = status.last_error || "—"; + document.getElementById("continuous-logs").textContent = (status.logs || []).join("\n") || "Логов пока нет."; +} + async function refreshDashboard() { const data = await api("/api/dashboard"); + const authData = await api("/api/auth"); + const continuousData = await api("/api/continuous"); state.dashboard = data; - renderAuth(data.auth); + renderAuthPanel(authData); renderSummary(data); renderChannels(data.channels); renderJobs(data.jobs); + renderContinuous(continuousData); } async function main() { @@ -167,6 +212,81 @@ async function main() { await refreshDashboard(); }); + document.getElementById("credentials-form").addEventListener("submit", async (event) => { + event.preventDefault(); + const apiId = document.getElementById("api-id-input").value.trim(); + const apiHash = document.getElementById("api-hash-input").value.trim(); + await api("/api/auth/credentials", { + method: "POST", + body: JSON.stringify({ api_id: apiId, api_hash: apiHash }), + }); + await refreshDashboard(); + }); + + document.getElementById("start-qr-btn").addEventListener("click", async (event) => { + setBusy(event.currentTarget, true); + try { + await api("/api/auth/qr/start", { + method: "POST", + body: JSON.stringify({}), + }); + await refreshDashboard(); + } finally { + setBusy(event.currentTarget, false); + } + }); + + document.getElementById("phone-form").addEventListener("submit", async (event) => { + event.preventDefault(); + const phone = document.getElementById("phone-input").value.trim(); + await api("/api/auth/phone/request", { + method: "POST", + body: JSON.stringify({ phone }), + }); + await refreshDashboard(); + }); + + document.getElementById("code-form").addEventListener("submit", async (event) => { + event.preventDefault(); + const code = document.getElementById("code-input").value.trim(); + await api("/api/auth/phone/submit", { + method: "POST", + body: JSON.stringify({ code }), + }); + event.currentTarget.reset(); + await refreshDashboard(); + }); + + document.getElementById("password-form").addEventListener("submit", async (event) => { + event.preventDefault(); + const password = document.getElementById("password-input").value.trim(); + await api("/api/auth/password", { + method: "POST", + body: JSON.stringify({ password }), + }); + event.currentTarget.reset(); + await refreshDashboard(); + }); + + document.getElementById("continuous-form").addEventListener("submit", async (event) => { + event.preventDefault(); + const enabled = document.getElementById("continuous-enabled").checked; + const intervalMinutes = document.getElementById("continuous-interval").value; + const runAllTracked = document.getElementById("continuous-all").checked; + const channelsRaw = document.getElementById("continuous-channels").value.trim(); + const channels = channelsRaw ? channelsRaw.split(",").map((item) => item.trim()).filter(Boolean) : []; + await api("/api/continuous", { + method: "POST", + body: JSON.stringify({ + enabled, + interval_minutes: intervalMinutes, + run_all_tracked: runAllTracked, + channels, + }), + }); + await refreshDashboard(); + }); + await refreshDashboard(); window.setInterval(refreshDashboard, 5000); } diff --git a/webui/index.html b/webui/index.html index 23cdf39..077b0c2 100644 --- a/webui/index.html +++ b/webui/index.html @@ -26,6 +26,39 @@

+
+
Telegram Login
+
+ + + +
+ +
+ +
+ + + +
+ + +
+ + + + +
+
Быстрые действия
@@ -92,6 +125,43 @@
+ +
+
+
+

Continuous Scraping

+

Периодически перескрапливает каналы и пишет лог прямо в панель.

+
+
+ +
+ + + + + +
+ +
+
Статус:
+
Последний цикл:
+
Последняя ошибка:
+
+ +

+        
diff --git a/webui/style.css b/webui/style.css index 9e152a7..0e681dc 100644 --- a/webui/style.css +++ b/webui/style.css @@ -186,6 +186,38 @@ button { flex-wrap: wrap; } +.stack-form { + display: grid; + gap: 8px; + margin-top: 12px; +} + +.auth-actions { + margin-top: 12px; +} + +.qr-wrap { + margin-top: 14px; + padding: 12px; + border: 1px solid var(--line); + border-radius: 8px; + background: #0d141c; +} + +.qr-wrap img { + display: block; + width: 100%; + max-width: 240px; + aspect-ratio: 1; + margin: 0 auto 10px; + border-radius: 8px; + background: white; +} + +.hidden { + display: none !important; +} + input { min-width: 160px; padding: 10px 12px; @@ -257,6 +289,17 @@ tbody tr:hover { white-space: pre-wrap; } +.continuous-form { + display: grid; + gap: 12px; +} + +.continuous-meta { + display: grid; + gap: 6px; + margin-top: 16px; +} + .viewer-shell { grid-template-columns: 320px minmax(0, 1fr); } diff --git a/webui_server.py b/webui_server.py index fdd742f..285774c 100644 --- a/webui_server.py +++ b/webui_server.py @@ -1,4 +1,5 @@ import asyncio +import base64 import contextlib import io import json @@ -16,7 +17,12 @@ from datetime import datetime, timezone from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Dict, List, Optional + +import qrcode +import qrcode.image.svg +from telethon import TelegramClient +from telethon.errors import SessionPasswordNeededError BASE_DIR = Path(__file__).resolve().parent @@ -25,6 +31,8 @@ WEBUI_DIR = BASE_DIR / "webui" STATE_FILE = DATA_DIR / "state.json" DEFAULT_HOST = os.environ.get("TELEGRAM_SCRAPER_HOST", "0.0.0.0") DEFAULT_PORT = int(os.environ.get("TELEGRAM_SCRAPER_PORT", "8080")) +SESSION_DIR = BASE_DIR / "session" +SESSION_DIR.mkdir(exist_ok=True) def utc_now_iso() -> str: @@ -284,6 +292,320 @@ class JobRunner: job.finished_at = utc_now_iso() +class TelegramAuthManager: + def __init__(self) -> None: + self.lock = threading.Lock() + self.loop = asyncio.new_event_loop() + self.thread = threading.Thread(target=self._run_loop, daemon=True) + self.thread.start() + self.client: Optional[TelegramClient] = None + self.qr_login = None + self.qr_wait_task = None + self.phone = None + self.phone_code_hash = None + self.state: Dict[str, Any] = { + "phase": "idle", + "status": "unknown", + "details": "", + "qr_url": None, + "qr_image": None, + "phone": None, + "updated_at": utc_now_iso(), + } + + def _run_loop(self) -> None: + asyncio.set_event_loop(self.loop) + self.loop.run_forever() + + def _run(self, coro): + future = asyncio.run_coroutine_threadsafe(coro, self.loop) + return future.result() + + def _set_state(self, **updates: Any) -> None: + with self.lock: + self.state.update(updates) + self.state["updated_at"] = utc_now_iso() + + def snapshot(self) -> Dict[str, Any]: + auth = auth_status() + with self.lock: + snapshot = dict(self.state) + snapshot["saved_credentials"] = { + "api_id": load_state().get("api_id"), + "api_hash_present": bool(load_state().get("api_hash")), + } + snapshot["auth_status"] = auth + return snapshot + + async def _get_client(self) -> TelegramClient: + state = load_state() + api_id = state.get("api_id") + api_hash = state.get("api_hash") + if not api_id or not api_hash: + raise RuntimeError("Save api_id and api_hash first.") + if self.client is None: + self.client = TelegramClient(str(SESSION_DIR / "session"), api_id, api_hash) + if not self.client.is_connected(): + await self.client.connect() + return self.client + + async def _is_authorized(self) -> bool: + client = await self._get_client() + return await client.is_user_authorized() + + def save_credentials(self, api_id: int, api_hash: str) -> Dict[str, Any]: + state = load_state() + state["api_id"] = int(api_id) + state["api_hash"] = api_hash.strip() + save_state(state) + self._set_state( + phase="credentials_saved", + status="ready_for_auth", + details="Credentials saved. You can login with QR or phone code.", + ) + return self.snapshot() + + def _make_qr_image(self, qr_url: str) -> str: + qr = qrcode.QRCode(border=1, box_size=8) + qr.add_data(qr_url) + qr.make(fit=True) + image = qr.make_image(image_factory=qrcode.image.svg.SvgPathImage) + buffer = io.BytesIO() + image.save(buffer) + encoded = base64.b64encode(buffer.getvalue()).decode("ascii") + return f"data:image/svg+xml;base64,{encoded}" + + async def _wait_for_qr_login(self) -> None: + try: + await self.qr_login.wait() + self._set_state( + phase="authorized", + status="authorized", + details="Telegram session is authorized.", + qr_url=None, + qr_image=None, + phone=None, + ) + except SessionPasswordNeededError: + self._set_state( + phase="password_required", + status="password_required", + details="Two-factor authentication is enabled. Enter your Telegram password.", + ) + except Exception as exc: + self._set_state( + phase="error", + status="error", + details=f"QR login failed: {exc}", + qr_url=None, + qr_image=None, + ) + + async def _start_qr_login(self) -> Dict[str, Any]: + client = await self._get_client() + if await client.is_user_authorized(): + self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.") + return self.snapshot() + self.qr_login = await client.qr_login() + qr_url = self.qr_login.url + self._set_state( + phase="qr_waiting", + status="qr_waiting", + details="Scan the QR code in Telegram: Settings -> Devices -> Scan QR.", + qr_url=qr_url, + qr_image=self._make_qr_image(qr_url), + ) + self.qr_wait_task = self.loop.create_task(self._wait_for_qr_login()) + return self.snapshot() + + def start_qr_login(self) -> Dict[str, Any]: + return self._run(self._start_qr_login()) + + async def _request_phone_code(self, phone: str) -> Dict[str, Any]: + client = await self._get_client() + if await client.is_user_authorized(): + self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.") + return self.snapshot() + sent = await client.send_code_request(phone) + self.phone = phone + self.phone_code_hash = sent.phone_code_hash + self._set_state( + phase="code_required", + status="code_required", + details=f"Code sent to {phone}. Enter it below.", + phone=phone, + ) + return self.snapshot() + + def request_phone_code(self, phone: str) -> Dict[str, Any]: + return self._run(self._request_phone_code(phone)) + + async def _submit_phone_code(self, code: str) -> Dict[str, Any]: + client = await self._get_client() + if not self.phone or not self.phone_code_hash: + raise RuntimeError("Request a phone code first.") + try: + await client.sign_in(phone=self.phone, code=code, phone_code_hash=self.phone_code_hash) + self._set_state( + phase="authorized", + status="authorized", + details="Telegram session is authorized.", + qr_url=None, + qr_image=None, + ) + except SessionPasswordNeededError: + self._set_state( + phase="password_required", + status="password_required", + details="Two-factor authentication is enabled. Enter your Telegram password.", + ) + return self.snapshot() + + def submit_phone_code(self, code: str) -> Dict[str, Any]: + return self._run(self._submit_phone_code(code)) + + async def _submit_password(self, password: str) -> Dict[str, Any]: + client = await self._get_client() + await client.sign_in(password=password) + self._set_state( + phase="authorized", + status="authorized", + details="Telegram session is authorized.", + qr_url=None, + qr_image=None, + ) + return self.snapshot() + + def submit_password(self, password: str) -> Dict[str, Any]: + return self._run(self._submit_password(password)) + + +class ContinuousScrapeManager: + def __init__(self) -> None: + self.lock = threading.Lock() + self.thread: Optional[threading.Thread] = None + self.stop_event = threading.Event() + self.config: Dict[str, Any] = { + "enabled": False, + "interval_minutes": 1, + "channels": [], + "run_all_tracked": True, + } + self.status: Dict[str, Any] = { + "running": False, + "last_started_at": None, + "last_finished_at": None, + "last_iteration_at": None, + "last_error": None, + "logs": [], + } + + def _log(self, message: str) -> None: + line = f"[{datetime.now().strftime('%H:%M:%S')}] {message}" + with self.lock: + logs = self.status.setdefault("logs", []) + logs.append(line) + if len(logs) > 200: + del logs[:-200] + + def snapshot(self) -> Dict[str, Any]: + with self.lock: + return { + "config": dict(self.config), + "status": { + **self.status, + "logs": list(self.status.get("logs", [])), + }, + } + + def update(self, enabled: bool, interval_minutes: int, channels: List[str], run_all_tracked: bool) -> Dict[str, Any]: + interval_minutes = max(1, int(interval_minutes)) + with self.lock: + self.config = { + "enabled": bool(enabled), + "interval_minutes": interval_minutes, + "channels": channels, + "run_all_tracked": bool(run_all_tracked), + } + if enabled: + self.start() + else: + self.stop() + return self.snapshot() + + def start(self) -> None: + with self.lock: + if self.status["running"]: + self._log("Continuous scraping is already running.") + return + self.stop_event.clear() + self.status["running"] = True + self.status["last_started_at"] = utc_now_iso() + self.status["last_error"] = None + self.thread = threading.Thread(target=self._run_loop, daemon=True) + self.thread.start() + self._log("Continuous scraping started.") + + def stop(self) -> None: + self.stop_event.set() + with self.lock: + was_running = self.status["running"] + self.status["running"] = False + if was_running: + self._log("Continuous scraping stop requested.") + + def _resolve_channels(self) -> List[str]: + state = load_state() + with self.lock: + run_all_tracked = self.config["run_all_tracked"] + configured = list(self.config["channels"]) + if run_all_tracked or not configured: + return list(state.get("channels", {}).keys()) + tracked = set(state.get("channels", {}).keys()) + return [channel for channel in configured if channel in tracked] + + def _run_loop(self) -> None: + while not self.stop_event.is_set(): + channels = self._resolve_channels() + interval_minutes = self.snapshot()["config"]["interval_minutes"] + + if not channels: + self._log("No channels configured for continuous scraping.") + else: + self._log(f"Starting iteration for {len(channels)} channel(s).") + buffer = io.StringIO() + try: + with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer): + run_job("scrape_selected", {"channels": channels}) + output = buffer.getvalue().strip() + if output: + for line in output.splitlines(): + self._log(line) + with self.lock: + self.status["last_iteration_at"] = utc_now_iso() + self.status["last_finished_at"] = utc_now_iso() + self.status["last_error"] = None + self._log("Iteration finished.") + except Exception as exc: + output = buffer.getvalue().strip() + if output: + for line in output.splitlines(): + self._log(line) + self._log(f"Iteration failed: {exc}") + with self.lock: + self.status["last_error"] = str(exc) + + sleep_seconds = max(5, interval_minutes * 60) + self._log(f"Sleeping for {interval_minutes} minute(s).") + interrupted = self.stop_event.wait(timeout=sleep_seconds) + if interrupted: + break + + with self.lock: + self.status["running"] = False + self._log("Continuous scraping stopped.") + + def import_scraper_class(): from telegram_scraper_with_forwarding import OptimizedTelegramScraper @@ -303,7 +625,9 @@ def run_job(job_type: str, payload: Dict[str, Any]) -> None: scraper = ScraperClass() initialized = await scraper.initialize_client(interactive=False) if not initialized: - raise RuntimeError("Telegram client is not ready. Check credentials and session.") + raise RuntimeError( + "Telegram client is not ready. Check credentials, session, and write access to /app/session." + ) try: if job_type == "scrape_channel": channel_id = payload["channel_id"] @@ -315,6 +639,11 @@ def run_job(job_type: str, payload: Dict[str, Any]) -> None: for channel_id in channels: offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0) await scraper.scrape_channel(channel_id, offset) + elif job_type == "scrape_selected": + channels = [str(channel_id) for channel_id in payload.get("channels", [])] + for channel_id in channels: + offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0) + await scraper.scrape_channel(channel_id, offset) elif job_type == "export_all": await scraper.export_data() elif job_type == "rescrape_media": @@ -402,6 +731,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.serve_media(relative) if path == "/api/dashboard": return self.send_json(dashboard_payload(self.app.job_runner)) + if path == "/api/auth": + return self.send_json(self.app.auth_manager.snapshot()) + if path == "/api/continuous": + return self.send_json(self.app.continuous_manager.snapshot()) if path == "/api/jobs": return self.send_json(self.app.job_runner.recent_jobs()) if path.startswith("/api/jobs/"): @@ -443,7 +776,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if path.startswith("/media/"): relative = path.removeprefix("/media/") return self.serve_media(relative, head_only=True) - if path in {"/api/dashboard", "/api/jobs", "/api/channels"} or path.startswith("/api/jobs/") or ( + if path in {"/api/dashboard", "/api/auth", "/api/continuous", "/api/jobs", "/api/channels"} or path.startswith("/api/jobs/") or ( path.startswith("/api/channels/") and path.endswith("/messages") ): self.send_response(HTTPStatus.OK) @@ -468,6 +801,70 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): ) return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) + if path == "/api/continuous": + try: + enabled = bool(body.get("enabled")) + interval_minutes = int(body.get("interval_minutes", 1)) + channels = [str(item).strip() for item in body.get("channels", []) if str(item).strip()] + run_all_tracked = bool(body.get("run_all_tracked", True)) + payload = self.app.continuous_manager.update( + enabled=enabled, + interval_minutes=interval_minutes, + channels=channels, + run_all_tracked=run_all_tracked, + ) + except Exception as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return self.send_json(payload) + + if path == "/api/auth/credentials": + api_id = body.get("api_id") + api_hash = str(body.get("api_hash", "")).strip() + if not api_id or not api_hash: + return self.send_error_json(HTTPStatus.BAD_REQUEST, "api_id and api_hash are required") + try: + payload = self.app.auth_manager.save_credentials(int(api_id), api_hash) + except Exception as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return self.send_json(payload) + + if path == "/api/auth/qr/start": + try: + payload = self.app.auth_manager.start_qr_login() + except Exception as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return self.send_json(payload) + + if path == "/api/auth/phone/request": + phone = str(body.get("phone", "")).strip() + if not phone: + return self.send_error_json(HTTPStatus.BAD_REQUEST, "phone is required") + try: + payload = self.app.auth_manager.request_phone_code(phone) + except Exception as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return self.send_json(payload) + + if path == "/api/auth/phone/submit": + code = str(body.get("code", "")).strip() + if not code: + return self.send_error_json(HTTPStatus.BAD_REQUEST, "code is required") + try: + payload = self.app.auth_manager.submit_phone_code(code) + except Exception as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return self.send_json(payload) + + if path == "/api/auth/password": + password = str(body.get("password", "")).strip() + if not password: + return self.send_error_json(HTTPStatus.BAD_REQUEST, "password is required") + try: + payload = self.app.auth_manager.submit_password(password) + except Exception as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return self.send_json(payload) + if path == "/api/channels/add": channel_id = str(body.get("channel_id", "")).strip() if not channel_id: @@ -605,6 +1002,8 @@ class TelegramScraperWebServer(ThreadingHTTPServer): def __init__(self, server_address: tuple[str, int]): super().__init__(server_address, TelegramScraperRequestHandler) self.job_runner = JobRunner() + self.auth_manager = TelegramAuthManager() + self.continuous_manager = ContinuousScrapeManager() def run_server(host: str = DEFAULT_HOST, port: int = DEFAULT_PORT) -> None: