forust/telegram-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Activity

Improve webui style and add api docs

Browse Source
This commit is contained in:
mr-forust
2026-05-24 22:18:41 +02:00
parent 6851922a9e
commit a78a708757
8 changed files with 1144 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui_server.py
+318 -18
View File
@@ -493,11 +493,11 @@ class TelegramAuthManager:
class ContinuousScrapeManager:
def __init__(self) -> None:
self.lock = threading.Lock()
self.lock = threading.RLock()
self.thread: Optional[threading.Thread] = None
self.stop_event = threading.Event()
self.config: Dict[str, Any] = {
"enabled": False,
"enabled": True,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
@@ -509,15 +509,22 @@ class ContinuousScrapeManager:
"last_iteration_at": None,
"last_error": None,
"logs": [],
"log_entries": [],
}
def _log(self, message: str) -> None:
def _log(self, message: str, level: str = "debug") -> None:
timestamp = utc_now_iso()
line = f"[{datetime.now().strftime('%H:%M:%S')}] {message}"
entry = {"timestamp": timestamp, "level": level, "message": message}
with self.lock:
logs = self.status.setdefault("logs", [])
log_entries = self.status.setdefault("log_entries", [])
logs.append(line)
if len(logs) > 200:
del logs[:-200]
log_entries.append(entry)
if len(logs) > 300:
del logs[:-300]
if len(log_entries) > 300:
del log_entries[:-300]
def snapshot(self) -> Dict[str, Any]:
with self.lock:
@@ -526,6 +533,7 @@ class ContinuousScrapeManager:
"status": {
**self.status,
"logs": list(self.status.get("logs", [])),
"log_entries": list(self.status.get("log_entries", [])),
},
}
@@ -553,7 +561,7 @@ class ContinuousScrapeManager:
def start(self) -> None:
with self.lock:
if self.status["running"]:
self._log("Continuous scraping is already running.")
self._log("Continuous scraping is already running.", "warn")
return
self.stop_event.clear()
self.status["running"] = True
@@ -561,7 +569,7 @@ class ContinuousScrapeManager:
self.status["last_error"] = None
self.thread = threading.Thread(target=self._run_loop, daemon=True)
self.thread.start()
self._log("Continuous scraping started.")
self._log("Continuous scraping started.", "info")
def stop(self) -> None:
self.stop_event.set()
@@ -569,14 +577,14 @@ class ContinuousScrapeManager:
was_running = self.status["running"]
self.status["running"] = False
if was_running:
self._log("Continuous scraping stop requested.")
self._log("Continuous scraping stop requested.", "warn")
def _resolve_channels(self) -> List[str]:
state = load_state()
with self.lock:
run_all_tracked = self.config["run_all_tracked"]
configured = list(self.config["channels"])
if run_all_tracked or not configured:
if run_all_tracked:
return list(state.get("channels", {}).keys())
tracked = set(state.get("channels", {}).keys())
return [channel for channel in configured if channel in tracked]
@@ -587,11 +595,9 @@ class ContinuousScrapeManager:
interval_minutes = self.snapshot()["config"]["interval_minutes"]
if not channels:
self._log("No channels configured for continuous scraping.")
self._log("No channels configured for continuous scraping.", "warn")
else:
self._log("=" * 50)
self._log(f"🚀 Starting iteration for {len(channels)} channel(s).")
self._log("-" * 50)
self._log(f"Starting iteration for {len(channels)} channel(s).", "info")
buffer = io.StringIO()
try:
with (
@@ -607,26 +613,25 @@ class ContinuousScrapeManager:
self.status["last_iteration_at"] = utc_now_iso()
self.status["last_finished_at"] = utc_now_iso()
self.status["last_error"] = None
self._log("-" * 50)
self._log("✅ Iteration finished.")
self._log("Iteration finished.", "success")
except Exception as exc:
output = buffer.getvalue().strip()
if output:
for line in output.splitlines():
self._log(line)
self._log(f"Iteration failed: {exc}")
self._log(f"Iteration failed: {exc}", "error")
with self.lock:
self.status["last_error"] = str(exc)
sleep_seconds = max(5, interval_minutes * 60)
self._log(f"💤 Sleeping for {interval_minutes} minute(s).")
self._log(f"Sleeping for {interval_minutes} minute(s).", "debug")
interrupted = self.stop_event.wait(timeout=sleep_seconds)
if interrupted:
break
with self.lock:
self.status["running"] = False
self._log("Continuous scraping stopped.")
self._log("Continuous scraping stopped.", "warn")
def import_scraper_class():
@@ -742,6 +747,287 @@ def dashboard_payload(job_runner: JobRunner) -> Dict[str, Any]:
}
def openapi_payload() -> Dict[str, Any]:
json_response = {
"200": {
"description": "JSON response",
"content": {"application/json": {"schema": {"type": "object"}}},
}
}
accepted_response = {
"202": {
"description": "Job accepted",
"content": {"application/json": {"schema": {"type": "object"}}},
}
}
error_response = {"400": {"description": "Bad request"}}
def json_body(properties: Dict[str, Any], required: Optional[List[str]] = None):
return {
"required": True,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": properties,
"required": required or [],
"example": {
key: value.get("example")
for key, value in properties.items()
if "example" in value
},
}
}
},
}
return {
"openapi": "3.0.3",
"info": {
"title": "Telegram Scraper Web UI API",
"version": "0.1.0",
"description": "Local endpoints used by the Telegram Scraper web interface.",
},
"servers": [{"url": "/"}],
"paths": {
"/api/dashboard": {
"get": {
"summary": "Dashboard snapshot",
"description": "Returns state summary, channels, auth status, and recent jobs.",
"responses": json_response,
}
},
"/api/auth": {
"get": {
"summary": "Authentication status",
"description": "Returns saved credential flags, login phase, and QR login data.",
"responses": json_response,
}
},
"/api/auth/credentials": {
"post": {
"summary": "Save Telegram API credentials",
"requestBody": json_body(
{
"api_id": {"type": "integer", "example": 123456},
"api_hash": {"type": "string", "example": "0123456789abcdef"},
},
["api_id", "api_hash"],
),
"responses": {**json_response, **error_response},
}
},
"/api/auth/qr/start": {
"post": {
"summary": "Start QR login",
"requestBody": json_body({}),
"responses": {**json_response, **error_response},
}
},
"/api/auth/phone/request": {
"post": {
"summary": "Request phone login code",
"requestBody": json_body(
{"phone": {"type": "string", "example": "+1234567890"}},
["phone"],
),
"responses": {**json_response, **error_response},
}
},
"/api/auth/phone/submit": {
"post": {
"summary": "Submit phone login code",
"requestBody": json_body(
{"code": {"type": "string", "example": "12345"}},
["code"],
),
"responses": {**json_response, **error_response},
}
},
"/api/auth/password": {
"post": {
"summary": "Submit 2FA password",
"requestBody": json_body(
{"password": {"type": "string", "example": "password"}},
["password"],
),
"responses": {**json_response, **error_response},
}
},
"/api/continuous": {
"get": {
"summary": "Continuous scraping status",
"responses": json_response,
},
"post": {
"summary": "Update continuous scraping",
"requestBody": json_body(
{
"enabled": {"type": "boolean", "example": True},
"interval_minutes": {"type": "integer", "example": 5},
"run_all_tracked": {"type": "boolean", "example": False},
"channels": {
"type": "array",
"items": {"type": "string"},
"example": ["-1001234567890"],
},
}
),
"responses": {**json_response, **error_response},
},
},
"/health/continuous": {
"get": {
"summary": "Continuous scraping health",
"responses": json_response,
}
},
"/api/channels": {
"get": {
"summary": "List tracked channels",
"responses": json_response,
}
},
"/api/channels/{channel_id}/messages": {
"get": {
"summary": "List channel messages",
"parameters": [
{
"name": "channel_id",
"in": "path",
"required": True,
"schema": {"type": "string"},
},
{
"name": "limit",
"in": "query",
"schema": {"type": "integer", "default": 120, "maximum": 300},
},
{
"name": "before",
"in": "query",
"schema": {"type": "integer"},
},
],
"responses": json_response,
}
},
"/api/channels/add": {
"post": {
"summary": "Track a channel",
"requestBody": json_body(
{
"channel_id": {"type": "string", "example": "-1001234567890"},
"name": {"type": "string", "example": "Research feed"},
},
["channel_id"],
),
"responses": {**json_response, **error_response},
}
},
"/api/channels/remove": {
"post": {
"summary": "Remove a tracked channel",
"requestBody": json_body(
{"channel_id": {"type": "string", "example": "-1001234567890"}},
["channel_id"],
),
"responses": {**json_response, **error_response},
}
},
"/api/settings/media": {
"post": {
"summary": "Toggle media scraping",
"requestBody": json_body(
{"value": {"type": "boolean", "example": True}},
["value"],
),
"responses": {**accepted_response, **error_response},
}
},
"/api/jobs": {
"get": {
"summary": "Recent jobs",
"responses": json_response,
}
},
"/api/jobs/{job_id}": {
"get": {
"summary": "Job details",
"parameters": [
{
"name": "job_id",
"in": "path",
"required": True,
"schema": {"type": "string"},
}
],
"responses": {**json_response, "404": {"description": "Job not found"}},
}
},
"/api/jobs/scrape": {
"post": {
"summary": "Scrape one channel or all tracked channels",
"requestBody": json_body(
{"channel_id": {"type": "string", "example": "-1001234567890"}}
),
"responses": accepted_response,
}
},
"/api/jobs/export": {
"post": {
"summary": "Export all tracked channels",
"requestBody": json_body({}),
"responses": accepted_response,
}
},
"/api/jobs/export-channel": {
"post": {
"summary": "Export one channel",
"requestBody": json_body(
{"channel_id": {"type": "string", "example": "-1001234567890"}},
["channel_id"],
),
"responses": {**accepted_response, **error_response},
}
},
"/api/jobs/rescrape-media": {
"post": {
"summary": "Rescrape media for a channel",
"requestBody": json_body(
{"channel_id": {"type": "string", "example": "-1001234567890"}},
["channel_id"],
),
"responses": {**accepted_response, **error_response},
}
},
"/api/jobs/fix-media": {
"post": {
"summary": "Fix missing media for a channel",
"requestBody": json_body(
{"channel_id": {"type": "string", "example": "-1001234567890"}},
["channel_id"],
),
"responses": {**accepted_response, **error_response},
}
},
"/api/jobs/refresh-dialogs": {
"post": {
"summary": "Refresh Telegram dialogs",
"requestBody": json_body({}),
"responses": accepted_response,
}
},
"/openapi.json": {
"get": {
"summary": "OpenAPI document",
"responses": json_response,
}
},
},
}
class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
server_version = "TelegramScraperWebUI/0.1"
@@ -760,12 +1046,18 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.serve_file(
WEBUI_DIR / "viewer.html", "text/html; charset=utf-8"
)
if path in {"/swagger", "/swigger", "/docs"}:
return self.serve_file(
WEBUI_DIR / "swagger.html", "text/html; charset=utf-8"
)
if path.startswith("/static/"):
relative = path.removeprefix("/static/")
return self.serve_static(relative)
if path.startswith("/media/"):
relative = path.removeprefix("/media/")
return self.serve_media(relative)
if path == "/openapi.json":
return self.send_json(openapi_payload())
if path == "/api/dashboard":
return self.send_json(dashboard_payload(self.app.job_runner))
if path == "/api/auth":
@@ -819,6 +1111,12 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.serve_file(
WEBUI_DIR / "viewer.html", "text/html; charset=utf-8", head_only=True
)
if path in {"/swagger", "/swigger", "/docs"}:
return self.serve_file(
WEBUI_DIR / "swagger.html",
"text/html; charset=utf-8",
head_only=True,
)
if path.startswith("/static/"):
relative = path.removeprefix("/static/")
return self.serve_static(relative, head_only=True)
@@ -834,6 +1132,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
"/api/jobs",
"/api/channels",
"/health/continuous",
"/openapi.json",
}
or path.startswith("/api/jobs/")
or (path.startswith("/api/channels/") and path.endswith("/messages"))
@@ -1098,6 +1397,7 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
self.job_runner = JobRunner()
self.auth_manager = TelegramAuthManager()
self.continuous_manager = ContinuousScrapeManager()
self.continuous_manager.start()
def run_server(host: str = DEFAULT_HOST, port: int = DEFAULT_PORT) -> None: