diff --git a/webui/app.js b/webui/app.js
index 8eff7fe..cc3937e 100644
--- a/webui/app.js
+++ b/webui/app.js
@@ -84,6 +84,14 @@ function renderChannels(channels) {
await refreshDashboard();
});
+ node.querySelector(".export-btn").addEventListener("click", async () => {
+ await api("/api/jobs/export-channel", {
+ method: "POST",
+ body: JSON.stringify({ channel_id: channel.channel_id }),
+ });
+ await refreshDashboard();
+ });
+
node.querySelector(".export-view-btn").addEventListener("click", () => {
window.location.href = `/viewer?channel=${encodeURIComponent(channel.channel_id)}`;
});
diff --git a/webui/index.html b/webui/index.html
index 8789f17..42d679c 100644
--- a/webui/index.html
+++ b/webui/index.html
@@ -194,8 +194,7 @@
|
-
-
+
diff --git a/webui_server.py b/webui_server.py
index 285774c..5e951f6 100644
--- a/webui_server.py
+++ b/webui_server.py
@@ -63,7 +63,9 @@ def channel_db_path(channel_id: str) -> Path:
return DATA_DIR / channel_id / f"{channel_id}.db"
-def guess_media_kind(media_path: Optional[str], media_type: Optional[str]) -> Optional[str]:
+def guess_media_kind(
+ media_path: Optional[str], media_type: Optional[str]
+) -> Optional[str]:
if not media_path:
return None
suffix = Path(media_path).suffix.lower()
@@ -129,10 +131,7 @@ def database_summary(channel_id: str) -> Dict[str, Any]:
def channel_display_name(state: Dict[str, Any], channel_id: str) -> str:
- return (
- state.get("channel_names", {}).get(channel_id)
- or channel_id
- )
+ return state.get("channel_names", {}).get(channel_id) or channel_id
def list_channels_snapshot() -> List[Dict[str, Any]]:
@@ -158,7 +157,9 @@ def list_channels_snapshot() -> List[Dict[str, Any]]:
return channels
-def load_messages(channel_id: str, limit: int = 120, before_message_id: Optional[int] = None) -> List[Dict[str, Any]]:
+def load_messages(
+ channel_id: str, limit: int = 120, before_message_id: Optional[int] = None
+) -> List[Dict[str, Any]]:
db_path = channel_db_path(channel_id)
if not db_path.exists():
return []
@@ -404,7 +405,11 @@ class TelegramAuthManager:
async def _start_qr_login(self) -> Dict[str, Any]:
client = await self._get_client()
if await client.is_user_authorized():
- self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.")
+ self._set_state(
+ phase="authorized",
+ status="authorized",
+ details="Telegram session is already authorized.",
+ )
return self.snapshot()
self.qr_login = await client.qr_login()
qr_url = self.qr_login.url
@@ -424,7 +429,11 @@ class TelegramAuthManager:
async def _request_phone_code(self, phone: str) -> Dict[str, Any]:
client = await self._get_client()
if await client.is_user_authorized():
- self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.")
+ self._set_state(
+ phase="authorized",
+ status="authorized",
+ details="Telegram session is already authorized.",
+ )
return self.snapshot()
sent = await client.send_code_request(phone)
self.phone = phone
@@ -445,7 +454,9 @@ class TelegramAuthManager:
if not self.phone or not self.phone_code_hash:
raise RuntimeError("Request a phone code first.")
try:
- await client.sign_in(phone=self.phone, code=code, phone_code_hash=self.phone_code_hash)
+ await client.sign_in(
+ phone=self.phone, code=code, phone_code_hash=self.phone_code_hash
+ )
self._set_state(
phase="authorized",
status="authorized",
@@ -518,7 +529,13 @@ class ContinuousScrapeManager:
},
}
- def update(self, enabled: bool, interval_minutes: int, channels: List[str], run_all_tracked: bool) -> Dict[str, Any]:
+ def update(
+ self,
+ enabled: bool,
+ interval_minutes: int,
+ channels: List[str],
+ run_all_tracked: bool,
+ ) -> Dict[str, Any]:
interval_minutes = max(1, int(interval_minutes))
with self.lock:
self.config = {
@@ -575,7 +592,10 @@ class ContinuousScrapeManager:
self._log(f"Starting iteration for {len(channels)} channel(s).")
buffer = io.StringIO()
try:
- with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer):
+ with (
+ contextlib.redirect_stdout(buffer),
+ contextlib.redirect_stderr(buffer),
+ ):
run_job("scrape_selected", {"channels": channels})
output = buffer.getvalue().strip()
if output:
@@ -637,15 +657,25 @@ def run_job(job_type: str, payload: Dict[str, Any]) -> None:
elif job_type == "scrape_all":
channels = list(scraper.state.get("channels", {}).keys())
for channel_id in channels:
- offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0)
+ offset = int(
+ scraper.state.get("channels", {}).get(channel_id, 0) or 0
+ )
await scraper.scrape_channel(channel_id, offset)
elif job_type == "scrape_selected":
- channels = [str(channel_id) for channel_id in payload.get("channels", [])]
+ channels = [
+ str(channel_id) for channel_id in payload.get("channels", [])
+ ]
for channel_id in channels:
- offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0)
+ offset = int(
+ scraper.state.get("channels", {}).get(channel_id, 0) or 0
+ )
await scraper.scrape_channel(channel_id, offset)
elif job_type == "export_all":
await scraper.export_data()
+ elif job_type == "export_channel":
+ channel_id = payload["channel_id"]
+ scraper.export_to_csv(channel_id)
+ scraper.export_to_json(channel_id)
elif job_type == "rescrape_media":
await scraper.rescrape_media(payload["channel_id"])
elif job_type == "fix_missing_media":
@@ -682,7 +712,9 @@ def auth_status() -> Dict[str, Any]:
status["session_ready"] = session_file.exists()
if status["has_api_credentials"] and status["session_ready"]:
status["status"] = "ready"
- status["details"] = "Scraping actions should be available after dependencies are installed."
+ status["details"] = (
+ "Scraping actions should be available after dependencies are installed."
+ )
elif status["has_api_credentials"]:
status["status"] = "needs_auth"
status["details"] = "API keys found, but Telegram session file is missing."
@@ -722,7 +754,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/":
return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8")
if path == "/viewer":
- return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8")
+ return self.serve_file(
+ WEBUI_DIR / "viewer.html", "text/html; charset=utf-8"
+ )
if path.startswith("/static/"):
relative = path.removeprefix("/static/")
return self.serve_static(relative)
@@ -735,6 +769,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_json(self.app.auth_manager.snapshot())
if path == "/api/continuous":
return self.send_json(self.app.continuous_manager.snapshot())
+ if path == "/health/continuous":
+ return self.send_json(self.app.continuous_manager.snapshot())
if path == "/api/jobs":
return self.send_json(self.app.job_runner.recent_jobs())
if path.startswith("/api/jobs/"):
@@ -753,9 +789,15 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
before_message_id = int(before[0]) if before else None
payload = {
"channel_id": channel_id,
- "messages": load_messages(channel_id, limit=limit, before_message_id=before_message_id),
+ "messages": load_messages(
+ channel_id, limit=limit, before_message_id=before_message_id
+ ),
"channel": next(
- (item for item in list_channels_snapshot() if item["channel_id"] == channel_id),
+ (
+ item
+ for item in list_channels_snapshot()
+ if item["channel_id"] == channel_id
+ ),
None,
),
}
@@ -767,17 +809,31 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
path = parsed.path
if path == "/":
- return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8", head_only=True)
+ return self.serve_file(
+ WEBUI_DIR / "index.html", "text/html; charset=utf-8", head_only=True
+ )
if path == "/viewer":
- return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8", head_only=True)
+ return self.serve_file(
+ WEBUI_DIR / "viewer.html", "text/html; charset=utf-8", head_only=True
+ )
if path.startswith("/static/"):
relative = path.removeprefix("/static/")
return self.serve_static(relative, head_only=True)
if path.startswith("/media/"):
relative = path.removeprefix("/media/")
return self.serve_media(relative, head_only=True)
- if path in {"/api/dashboard", "/api/auth", "/api/continuous", "/api/jobs", "/api/channels"} or path.startswith("/api/jobs/") or (
- path.startswith("/api/channels/") and path.endswith("/messages")
+ if (
+ path
+ in {
+ "/api/dashboard",
+ "/api/auth",
+ "/api/continuous",
+ "/api/jobs",
+ "/api/channels",
+ "/health/continuous",
+ }
+ or path.startswith("/api/jobs/")
+ or (path.startswith("/api/channels/") and path.endswith("/messages"))
):
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "application/json; charset=utf-8")
@@ -805,7 +861,11 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
try:
enabled = bool(body.get("enabled"))
interval_minutes = int(body.get("interval_minutes", 1))
- channels = [str(item).strip() for item in body.get("channels", []) if str(item).strip()]
+ channels = [
+ str(item).strip()
+ for item in body.get("channels", [])
+ if str(item).strip()
+ ]
run_all_tracked = bool(body.get("run_all_tracked", True))
payload = self.app.continuous_manager.update(
enabled=enabled,
@@ -821,7 +881,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
api_id = body.get("api_id")
api_hash = str(body.get("api_hash", "")).strip()
if not api_id or not api_hash:
- return self.send_error_json(HTTPStatus.BAD_REQUEST, "api_id and api_hash are required")
+ return self.send_error_json(
+ HTTPStatus.BAD_REQUEST, "api_id and api_hash are required"
+ )
try:
payload = self.app.auth_manager.save_credentials(int(api_id), api_hash)
except Exception as exc:
@@ -858,7 +920,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/api/auth/password":
password = str(body.get("password", "")).strip()
if not password:
- return self.send_error_json(HTTPStatus.BAD_REQUEST, "password is required")
+ return self.send_error_json(
+ HTTPStatus.BAD_REQUEST, "password is required"
+ )
try:
payload = self.app.auth_manager.submit_password(password)
except Exception as exc:
@@ -868,12 +932,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/api/channels/add":
channel_id = str(body.get("channel_id", "")).strip()
if not channel_id:
- return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+ return self.send_error_json(
+ HTTPStatus.BAD_REQUEST, "channel_id is required"
+ )
state = load_state()
if channel_id not in state["channels"]:
state["channels"][channel_id] = 0
if body.get("name"):
- state.setdefault("channel_names", {})[channel_id] = str(body["name"]).strip()
+ state.setdefault("channel_names", {})[channel_id] = str(
+ body["name"]
+ ).strip()
save_state(state)
return self.send_json({"ok": True, "channel_id": channel_id})
@@ -909,10 +977,25 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
+ if path == "/api/jobs/export-channel":
+ channel_id = str(body.get("channel_id", "")).strip()
+ if not channel_id:
+ return self.send_error_json(
+ HTTPStatus.BAD_REQUEST, "channel_id is required"
+ )
+ job = self.app.job_runner.create_job(
+ "export_channel",
+ f"Export channel {channel_id}",
+ {"channel_id": channel_id},
+ )
+ return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
+
if path == "/api/jobs/rescrape-media":
channel_id = str(body.get("channel_id", "")).strip()
if not channel_id:
- return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+ return self.send_error_json(
+ HTTPStatus.BAD_REQUEST, "channel_id is required"
+ )
job = self.app.job_runner.create_job(
"rescrape_media",
f"Rescrape media for {channel_id}",
@@ -923,7 +1006,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/api/jobs/fix-media":
channel_id = str(body.get("channel_id", "")).strip()
if not channel_id:
- return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+ return self.send_error_json(
+ HTTPStatus.BAD_REQUEST, "channel_id is required"
+ )
job = self.app.job_runner.create_job(
"fix_missing_media",
f"Fix missing media for {channel_id}",
@@ -959,7 +1044,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
if not file_path.exists() or not file_path.is_file():
return self.send_error_json(HTTPStatus.NOT_FOUND, "File not found")
- mime_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
+ mime_type = (
+ mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
+ )
return self.serve_file(file_path, mime_type, head_only=head_only)
def serve_media(self, relative_path: str, head_only: bool = False) -> None:
@@ -971,10 +1058,14 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_error_json(HTTPStatus.FORBIDDEN, "Forbidden")
if not file_path.exists() or not file_path.is_file():
return self.send_error_json(HTTPStatus.NOT_FOUND, "Media file not found")
- mime_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
+ mime_type = (
+ mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
+ )
return self.serve_file(file_path, mime_type, head_only=head_only)
- def serve_file(self, file_path: Path, content_type: str, head_only: bool = False) -> None:
+ def serve_file(
+ self, file_path: Path, content_type: str, head_only: bool = False
+ ) -> None:
data = file_path.read_bytes()
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", content_type)
|