fix(server): harden auth, SSE, state, scraping

- Fix SSE streams not terminating on successful jobs (C-2)
- Anchor data/session paths to BASE_DIR instead of CWD (C-3)
- Guard TelegramAuthManager state with RLock (H-1)
- Replace millisecond job ids with uuid4 (H-2)
- Always redact api_id/api_hash on export, drop include_secrets (H-3)
- Enforce JSON content-type + same-origin on mutating requests (H-4)
- Rate-limit auth attempts and phone-code requests (H-5)
- Deep-copy StateStore.load() on all paths (H-6)
- Cap FloodWait retries in forward_message (H-7)
- De-duplicate forwarding handler registration (H-8)
- Validate continuous channels at ingest, join scrape thread on account
  removal, fix refresh_config status under lock, cap SSE streams and
  JSON body size (M-5, M-6, M-17)
- Add regression tests (33 passing) and REVIEW.md

telegram_scraper_with_forwarding.py

@@ -2,6 +2,7 @@ import sqlite3
 import json
 import csv
 import asyncio
+import logging
 import time
 import sys
 import warnings
@@ -27,6 +28,9 @@ from app_state import (
     get_account_store,
 )
 
+BASE_DIR = Path(__file__).resolve().parent
+logger = logging.getLogger(__name__)
+
 warnings.filterwarnings(
     "ignore", message="Using async sessions support is an experimental feature"
 )
@@ -90,16 +94,18 @@ def _ensure_session_wal(session_path: str) -> None:
 
 
 class OptimizedTelegramScraper:
-    def __init__(self, account_id: Optional[str] = None):
+    def __init__(self, account_id: Optional[str] = None, base_dir: Optional[Path] = None):
         self.account_id = account_id
-        self.SESSION_DIR = Path("session")
+        base_dir = base_dir or BASE_DIR
+        self.BASE_DIR = base_dir
+        self.SESSION_DIR = base_dir / "session"
         self.SESSION_DIR.mkdir(exist_ok=True)
 
         if account_id:
-            self.DATA_DIR = Path("data") / "accounts" / account_id
-            self.state_store = get_account_store(Path("data"), account_id)
+            self.DATA_DIR = base_dir / "data" / "accounts" / account_id
+            self.state_store = get_account_store(base_dir / "data", account_id)
         else:
-            self.DATA_DIR = Path("data")
+            self.DATA_DIR = base_dir / "data"
             self.state_store = StateStore(self.DATA_DIR / "state.json")
 
         self.DATA_DIR.mkdir(parents=True, exist_ok=True)
@@ -210,6 +216,22 @@ class OptimizedTelegramScraper:
         columns = {row[1] for row in cursor.fetchall()}
 
         migrations = []
+        if "sender_id" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN sender_id INTEGER")
+        if "first_name" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN first_name TEXT")
+        if "last_name" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN last_name TEXT")
+        if "username" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN username TEXT")
+        if "message" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN message TEXT")
+        if "media_type" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN media_type TEXT")
+        if "media_path" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN media_path TEXT")
+        if "reply_to" not in columns:
+            migrations.append("ALTER TABLE messages ADD COLUMN reply_to INTEGER")
         if "post_author" not in columns:
             migrations.append("ALTER TABLE messages ADD COLUMN post_author TEXT")
         if "views" not in columns:
@@ -222,8 +244,8 @@ class OptimizedTelegramScraper:
         for migration in migrations:
             try:
                 conn.execute(migration)
-            except Exception:
-                pass
+            except Exception as e:
+                logger.warning("Migration failed for %s: %s", migration, e)
 
         if migrations:
             conn.commit()
@@ -353,7 +375,7 @@ class OptimizedTelegramScraper:
         return False
 
     async def forward_message(
-        self, message, rule: ForwardingRule, source_channel_id: int = None
+        self, message, rule: ForwardingRule, source_channel_id: int = None, _retry: int = 0
     ):
         try:
             dest_entity = await self._resolve_entity(rule.destination_channel)
@@ -393,9 +415,12 @@ class OptimizedTelegramScraper:
 
             return True
         except FloodWaitError as e:
+            if _retry >= 3:
+                print(f"  Failed to forward message {message.id}: FloodWait retry limit exceeded")
+                return False
             print(f"  Rate limited, waiting {e.seconds}s...")
             await asyncio.sleep(e.seconds)
-            return await self.forward_message(message, rule, source_channel_id)
+            return await self.forward_message(message, rule, source_channel_id, _retry=_retry + 1)
         except Exception as e:
             print(f"  Failed to forward message {message.id}: {e}")
             return False
@@ -429,6 +454,11 @@ class OptimizedTelegramScraper:
             print("No valid source channels")
             return False
 
+        # Unregister a previously installed handler so it is never registered twice.
+        if self.forwarding_handler is not None:
+            self.client.remove_event_handler(self.forwarding_handler)
+            self.forwarding_handler = None
+
         @self.client.on(
             events.NewMessage(chats=source_channels, incoming=True, outgoing=True)
         )