From be02eed3784f0bd1e4a49f6d8e004b08aac38f66 Mon Sep 17 00:00:00 2001 From: mr-forust Date: Sat, 25 Apr 2026 22:12:14 +0200 Subject: [PATCH 1/3] Add export button for channel data and update API integration Co-authored-by: Copilot --- webui/app.js | 8 +++ webui/index.html | 3 +- webui_server.py | 155 +++++++++++++++++++++++++++++++++++++---------- 3 files changed, 132 insertions(+), 34 deletions(-) diff --git a/webui/app.js b/webui/app.js index 8eff7fe..cc3937e 100644 --- a/webui/app.js +++ b/webui/app.js @@ -84,6 +84,14 @@ function renderChannels(channels) { await refreshDashboard(); }); + node.querySelector(".export-btn").addEventListener("click", async () => { + await api("/api/jobs/export-channel", { + method: "POST", + body: JSON.stringify({ channel_id: channel.channel_id }), + }); + await refreshDashboard(); + }); + node.querySelector(".export-view-btn").addEventListener("click", () => { window.location.href = `/viewer?channel=${encodeURIComponent(channel.channel_id)}`; }); diff --git a/webui/index.html b/webui/index.html index 8789f17..42d679c 100644 --- a/webui/index.html +++ b/webui/index.html @@ -194,8 +194,7 @@
- - +
diff --git a/webui_server.py b/webui_server.py index 285774c..5e951f6 100644 --- a/webui_server.py +++ b/webui_server.py @@ -63,7 +63,9 @@ def channel_db_path(channel_id: str) -> Path: return DATA_DIR / channel_id / f"{channel_id}.db" -def guess_media_kind(media_path: Optional[str], media_type: Optional[str]) -> Optional[str]: +def guess_media_kind( + media_path: Optional[str], media_type: Optional[str] +) -> Optional[str]: if not media_path: return None suffix = Path(media_path).suffix.lower() @@ -129,10 +131,7 @@ def database_summary(channel_id: str) -> Dict[str, Any]: def channel_display_name(state: Dict[str, Any], channel_id: str) -> str: - return ( - state.get("channel_names", {}).get(channel_id) - or channel_id - ) + return state.get("channel_names", {}).get(channel_id) or channel_id def list_channels_snapshot() -> List[Dict[str, Any]]: @@ -158,7 +157,9 @@ def list_channels_snapshot() -> List[Dict[str, Any]]: return channels -def load_messages(channel_id: str, limit: int = 120, before_message_id: Optional[int] = None) -> List[Dict[str, Any]]: +def load_messages( + channel_id: str, limit: int = 120, before_message_id: Optional[int] = None +) -> List[Dict[str, Any]]: db_path = channel_db_path(channel_id) if not db_path.exists(): return [] @@ -404,7 +405,11 @@ class TelegramAuthManager: async def _start_qr_login(self) -> Dict[str, Any]: client = await self._get_client() if await client.is_user_authorized(): - self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.") + self._set_state( + phase="authorized", + status="authorized", + details="Telegram session is already authorized.", + ) return self.snapshot() self.qr_login = await client.qr_login() qr_url = self.qr_login.url @@ -424,7 +429,11 @@ class TelegramAuthManager: async def _request_phone_code(self, phone: str) -> Dict[str, Any]: client = await self._get_client() if await client.is_user_authorized(): - self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.") + self._set_state( + phase="authorized", + status="authorized", + details="Telegram session is already authorized.", + ) return self.snapshot() sent = await client.send_code_request(phone) self.phone = phone @@ -445,7 +454,9 @@ class TelegramAuthManager: if not self.phone or not self.phone_code_hash: raise RuntimeError("Request a phone code first.") try: - await client.sign_in(phone=self.phone, code=code, phone_code_hash=self.phone_code_hash) + await client.sign_in( + phone=self.phone, code=code, phone_code_hash=self.phone_code_hash + ) self._set_state( phase="authorized", status="authorized", @@ -518,7 +529,13 @@ class ContinuousScrapeManager: }, } - def update(self, enabled: bool, interval_minutes: int, channels: List[str], run_all_tracked: bool) -> Dict[str, Any]: + def update( + self, + enabled: bool, + interval_minutes: int, + channels: List[str], + run_all_tracked: bool, + ) -> Dict[str, Any]: interval_minutes = max(1, int(interval_minutes)) with self.lock: self.config = { @@ -575,7 +592,10 @@ class ContinuousScrapeManager: self._log(f"Starting iteration for {len(channels)} channel(s).") buffer = io.StringIO() try: - with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer): + with ( + contextlib.redirect_stdout(buffer), + contextlib.redirect_stderr(buffer), + ): run_job("scrape_selected", {"channels": channels}) output = buffer.getvalue().strip() if output: @@ -637,15 +657,25 @@ def run_job(job_type: str, payload: Dict[str, Any]) -> None: elif job_type == "scrape_all": channels = list(scraper.state.get("channels", {}).keys()) for channel_id in channels: - offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0) + offset = int( + scraper.state.get("channels", {}).get(channel_id, 0) or 0 + ) await scraper.scrape_channel(channel_id, offset) elif job_type == "scrape_selected": - channels = [str(channel_id) for channel_id in payload.get("channels", [])] + channels = [ + str(channel_id) for channel_id in payload.get("channels", []) + ] for channel_id in channels: - offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0) + offset = int( + scraper.state.get("channels", {}).get(channel_id, 0) or 0 + ) await scraper.scrape_channel(channel_id, offset) elif job_type == "export_all": await scraper.export_data() + elif job_type == "export_channel": + channel_id = payload["channel_id"] + scraper.export_to_csv(channel_id) + scraper.export_to_json(channel_id) elif job_type == "rescrape_media": await scraper.rescrape_media(payload["channel_id"]) elif job_type == "fix_missing_media": @@ -682,7 +712,9 @@ def auth_status() -> Dict[str, Any]: status["session_ready"] = session_file.exists() if status["has_api_credentials"] and status["session_ready"]: status["status"] = "ready" - status["details"] = "Scraping actions should be available after dependencies are installed." + status["details"] = ( + "Scraping actions should be available after dependencies are installed." + ) elif status["has_api_credentials"]: status["status"] = "needs_auth" status["details"] = "API keys found, but Telegram session file is missing." @@ -722,7 +754,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if path == "/": return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8") if path == "/viewer": - return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8") + return self.serve_file( + WEBUI_DIR / "viewer.html", "text/html; charset=utf-8" + ) if path.startswith("/static/"): relative = path.removeprefix("/static/") return self.serve_static(relative) @@ -735,6 +769,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(self.app.auth_manager.snapshot()) if path == "/api/continuous": return self.send_json(self.app.continuous_manager.snapshot()) + if path == "/health/continuous": + return self.send_json(self.app.continuous_manager.snapshot()) if path == "/api/jobs": return self.send_json(self.app.job_runner.recent_jobs()) if path.startswith("/api/jobs/"): @@ -753,9 +789,15 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): before_message_id = int(before[0]) if before else None payload = { "channel_id": channel_id, - "messages": load_messages(channel_id, limit=limit, before_message_id=before_message_id), + "messages": load_messages( + channel_id, limit=limit, before_message_id=before_message_id + ), "channel": next( - (item for item in list_channels_snapshot() if item["channel_id"] == channel_id), + ( + item + for item in list_channels_snapshot() + if item["channel_id"] == channel_id + ), None, ), } @@ -767,17 +809,31 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): path = parsed.path if path == "/": - return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8", head_only=True) + return self.serve_file( + WEBUI_DIR / "index.html", "text/html; charset=utf-8", head_only=True + ) if path == "/viewer": - return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8", head_only=True) + return self.serve_file( + WEBUI_DIR / "viewer.html", "text/html; charset=utf-8", head_only=True + ) if path.startswith("/static/"): relative = path.removeprefix("/static/") return self.serve_static(relative, head_only=True) if path.startswith("/media/"): relative = path.removeprefix("/media/") return self.serve_media(relative, head_only=True) - if path in {"/api/dashboard", "/api/auth", "/api/continuous", "/api/jobs", "/api/channels"} or path.startswith("/api/jobs/") or ( - path.startswith("/api/channels/") and path.endswith("/messages") + if ( + path + in { + "/api/dashboard", + "/api/auth", + "/api/continuous", + "/api/jobs", + "/api/channels", + "/health/continuous", + } + or path.startswith("/api/jobs/") + or (path.startswith("/api/channels/") and path.endswith("/messages")) ): self.send_response(HTTPStatus.OK) self.send_header("Content-Type", "application/json; charset=utf-8") @@ -805,7 +861,11 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): try: enabled = bool(body.get("enabled")) interval_minutes = int(body.get("interval_minutes", 1)) - channels = [str(item).strip() for item in body.get("channels", []) if str(item).strip()] + channels = [ + str(item).strip() + for item in body.get("channels", []) + if str(item).strip() + ] run_all_tracked = bool(body.get("run_all_tracked", True)) payload = self.app.continuous_manager.update( enabled=enabled, @@ -821,7 +881,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): api_id = body.get("api_id") api_hash = str(body.get("api_hash", "")).strip() if not api_id or not api_hash: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "api_id and api_hash are required") + return self.send_error_json( + HTTPStatus.BAD_REQUEST, "api_id and api_hash are required" + ) try: payload = self.app.auth_manager.save_credentials(int(api_id), api_hash) except Exception as exc: @@ -858,7 +920,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if path == "/api/auth/password": password = str(body.get("password", "")).strip() if not password: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "password is required") + return self.send_error_json( + HTTPStatus.BAD_REQUEST, "password is required" + ) try: payload = self.app.auth_manager.submit_password(password) except Exception as exc: @@ -868,12 +932,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if path == "/api/channels/add": channel_id = str(body.get("channel_id", "")).strip() if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + return self.send_error_json( + HTTPStatus.BAD_REQUEST, "channel_id is required" + ) state = load_state() if channel_id not in state["channels"]: state["channels"][channel_id] = 0 if body.get("name"): - state.setdefault("channel_names", {})[channel_id] = str(body["name"]).strip() + state.setdefault("channel_names", {})[channel_id] = str( + body["name"] + ).strip() save_state(state) return self.send_json({"ok": True, "channel_id": channel_id}) @@ -909,10 +977,25 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): ) return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) + if path == "/api/jobs/export-channel": + channel_id = str(body.get("channel_id", "")).strip() + if not channel_id: + return self.send_error_json( + HTTPStatus.BAD_REQUEST, "channel_id is required" + ) + job = self.app.job_runner.create_job( + "export_channel", + f"Export channel {channel_id}", + {"channel_id": channel_id}, + ) + return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) + if path == "/api/jobs/rescrape-media": channel_id = str(body.get("channel_id", "")).strip() if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + return self.send_error_json( + HTTPStatus.BAD_REQUEST, "channel_id is required" + ) job = self.app.job_runner.create_job( "rescrape_media", f"Rescrape media for {channel_id}", @@ -923,7 +1006,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if path == "/api/jobs/fix-media": channel_id = str(body.get("channel_id", "")).strip() if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + return self.send_error_json( + HTTPStatus.BAD_REQUEST, "channel_id is required" + ) job = self.app.job_runner.create_job( "fix_missing_media", f"Fix missing media for {channel_id}", @@ -959,7 +1044,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden") if not file_path.exists() or not file_path.is_file(): return self.send_error_json(HTTPStatus.NOT_FOUND, "File not found") - mime_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream" + mime_type = ( + mimetypes.guess_type(str(file_path))[0] or "application/octet-stream" + ) return self.serve_file(file_path, mime_type, head_only=head_only) def serve_media(self, relative_path: str, head_only: bool = False) -> None: @@ -971,10 +1058,14 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden") if not file_path.exists() or not file_path.is_file(): return self.send_error_json(HTTPStatus.NOT_FOUND, "Media file not found") - mime_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream" + mime_type = ( + mimetypes.guess_type(str(file_path))[0] or "application/octet-stream" + ) return self.serve_file(file_path, mime_type, head_only=head_only) - def serve_file(self, file_path: Path, content_type: str, head_only: bool = False) -> None: + def serve_file( + self, file_path: Path, content_type: str, head_only: bool = False + ) -> None: data = file_path.read_bytes() self.send_response(HTTPStatus.OK) self.send_header("Content-Type", content_type) From cc99f752b301ae0cff235505b31cb3e082d526f6 Mon Sep 17 00:00:00 2001 From: mr-forust Date: Sun, 26 Apr 2026 21:45:12 +0200 Subject: [PATCH 2/3] Change exposed port to 7887 --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 1968309..fa8323a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,7 +7,7 @@ services: user: 1000:1000 restart: unless-stopped ports: - - "8080:8080" + - "7887:8080" volumes: - ./data:/app/data - ./session:/app/session From ff000e88d43ef17fc7c0936d0420e071fd3795a4 Mon Sep 17 00:00:00 2001 From: mr-forust Date: Sun, 26 Apr 2026 23:21:26 +0200 Subject: [PATCH 3/3] Enchance weblog --- telegram_scraper_with_forwarding.py | 15 ++++++++------- webui_server.py | 9 ++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/telegram_scraper_with_forwarding.py b/telegram_scraper_with_forwarding.py index 4704064..dbf7240 100644 --- a/telegram_scraper_with_forwarding.py +++ b/telegram_scraper_with_forwarding.py @@ -829,7 +829,7 @@ class OptimizedTelegramScraper: self.state['channels'][channel] = last_message_id self.save_state() - print(f"\nCompleted scraping channel {channel}") + print(f"Completed scraping channel {channel}") except Exception as e: print(f"Error with channel {channel}: {e}") @@ -843,10 +843,10 @@ class OptimizedTelegramScraper: channel_name = self.state.get('channel_names', {}).get(channel, 'Unknown') if not message_ids: - print(f"No media files to reprocess for {channel_name} (ID: {channel})") + # print(f"No media files to reprocess for {channel_name} (ID: {channel})") return - print(f"📥 Reprocessing {len(message_ids)} media files for {channel_name} (ID: {channel})") + # print(f"📥 Reprocessing {len(message_ids)} media files for {channel_name} (ID: {channel})") try: entity = await self._resolve_entity(channel) @@ -881,10 +881,10 @@ class OptimizedTelegramScraper: filled_length = int(bar_length * completed_media // len(message_ids)) bar = '█' * filled_length + '░' * (bar_length - filled_length) - sys.stdout.write(f"\r🔄 Rescrape: [{bar}] {progress:.1f}% ({completed_media}/{len(message_ids)})") - sys.stdout.flush() + # sys.stdout.write(f"\r🔄 Rescrape: [{bar}] {progress:.1f}% ({completed_media}/{len(message_ids)})") + # sys.stdout.flush() - print(f"\n✅ Media reprocessing complete! ({successful_downloads}/{len(message_ids)} successful)") + # print(f"\n✅ Media reprocessing complete! ({successful_downloads}/{len(message_ids)} successful)") except Exception as e: print(f"Error reprocessing media: {e}") @@ -1306,7 +1306,8 @@ class OptimizedTelegramScraper: await self.client.disconnect() return False else: - print("✅ Already authenticated!") + pass + # print("✅ Already authenticated!") return True diff --git a/webui_server.py b/webui_server.py index 5e951f6..9fa86d2 100644 --- a/webui_server.py +++ b/webui_server.py @@ -589,7 +589,9 @@ class ContinuousScrapeManager: if not channels: self._log("No channels configured for continuous scraping.") else: - self._log(f"Starting iteration for {len(channels)} channel(s).") + self._log("=" * 50) + self._log(f"🚀 Starting iteration for {len(channels)} channel(s).") + self._log("-" * 50) buffer = io.StringIO() try: with ( @@ -605,7 +607,8 @@ class ContinuousScrapeManager: self.status["last_iteration_at"] = utc_now_iso() self.status["last_finished_at"] = utc_now_iso() self.status["last_error"] = None - self._log("Iteration finished.") + self._log("-" * 50) + self._log("✅ Iteration finished.") except Exception as exc: output = buffer.getvalue().strip() if output: @@ -616,7 +619,7 @@ class ContinuousScrapeManager: self.status["last_error"] = str(exc) sleep_seconds = max(5, interval_minutes * 60) - self._log(f"Sleeping for {interval_minutes} minute(s).") + self._log(f"💤 Sleeping for {interval_minutes} minute(s).") interrupted = self.stop_event.wait(timeout=sleep_seconds) if interrupted: break