forust
a2468a2a2c
- Fix SSE streams not terminating on successful jobs (C-2) - Anchor data/session paths to BASE_DIR instead of CWD (C-3) - Guard TelegramAuthManager state with RLock (H-1) - Replace millisecond job ids with uuid4 (H-2) - Always redact api_id/api_hash on export, drop include_secrets (H-3) - Enforce JSON content-type + same-origin on mutating requests (H-4) - Rate-limit auth attempts and phone-code requests (H-5) - Deep-copy StateStore.load() on all paths (H-6) - Cap FloodWait retries in forward_message (H-7) - De-duplicate forwarding handler registration (H-8) - Validate continuous channels at ingest, join scrape thread on account removal, fix refresh_config status under lock, cap SSE streams and JSON body size (M-5, M-6, M-17) - Add regression tests (33 passing) and REVIEW.md
87 lines
3.3 KiB
Python
87 lines
3.3 KiB
Python
import asyncio
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from app_state import StateStore
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent
|
|
|
|
|
|
class ScraperJobService:
|
|
def __init__(self, state_store: StateStore):
|
|
self.state_store = state_store
|
|
|
|
def run(self, job_type: str, payload: Dict[str, Any]) -> None:
|
|
if job_type == "set_scrape_media":
|
|
value = bool(payload["value"])
|
|
|
|
def mutate(state: Dict[str, Any]) -> None:
|
|
state["scrape_media"] = value
|
|
|
|
self.state_store.update(mutate)
|
|
logger.info("Media scraping set to %s", value)
|
|
return
|
|
|
|
asyncio.run(self._run_async(job_type, payload))
|
|
|
|
async def _run_async(self, job_type: str, payload: Dict[str, Any]) -> None:
|
|
# Extract account_id from payload, default to None (legacy)
|
|
account_id: Optional[str] = payload.get("account_id")
|
|
ScraperClass = self._import_scraper_class()
|
|
scraper = ScraperClass(account_id=account_id, base_dir=BASE_DIR)
|
|
|
|
if account_id:
|
|
from app_state import load_account
|
|
acc_state = load_account(self.state_store.path.parent, account_id)
|
|
# Load per-account state into scraper state
|
|
scraper.state = acc_state
|
|
|
|
initialized = await scraper.initialize_client(interactive=False)
|
|
if not initialized:
|
|
raise RuntimeError(
|
|
"Telegram client is not ready. Check credentials, session, and write access."
|
|
)
|
|
try:
|
|
if job_type == "scrape_channel":
|
|
await self._scrape_channels(scraper, [payload["channel_id"]])
|
|
elif job_type == "scrape_all":
|
|
await self._scrape_channels(
|
|
scraper, list(scraper.state.get("channels", {}).keys())
|
|
)
|
|
elif job_type == "scrape_selected":
|
|
await self._scrape_channels(
|
|
scraper,
|
|
[str(channel_id) for channel_id in payload.get("channels", [])],
|
|
)
|
|
elif job_type == "export_all":
|
|
await scraper.export_data()
|
|
elif job_type == "export_channel":
|
|
channel_id = payload["channel_id"]
|
|
scraper.export_to_csv(channel_id)
|
|
scraper.export_to_json(channel_id)
|
|
elif job_type == "rescrape_media":
|
|
await scraper.rescrape_media(payload["channel_id"])
|
|
elif job_type == "fix_missing_media":
|
|
await scraper.fix_missing_media(payload["channel_id"])
|
|
elif job_type == "refresh_dialogs":
|
|
await scraper.list_channels()
|
|
else:
|
|
raise RuntimeError(f"Unsupported job type: {job_type}")
|
|
finally:
|
|
scraper.close_db_connections()
|
|
if scraper.client:
|
|
await scraper.client.disconnect()
|
|
|
|
async def _scrape_channels(self, scraper, channels: List[str]) -> None:
|
|
for channel_id in channels:
|
|
offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0)
|
|
await scraper.scrape_channel(channel_id, offset)
|
|
|
|
def _import_scraper_class(self):
|
|
from telegram_scraper_with_forwarding import OptimizedTelegramScraper
|
|
|
|
return OptimizedTelegramScraper
|