diff --git a/app_state.py b/app_state.py index 3d6a910..5a84e21 100644 --- a/app_state.py +++ b/app_state.py @@ -25,7 +25,7 @@ ACCOUNT_DEFAULTS: Dict[str, Any] = { "scrape_media": True, "forwarding_rules": [], "continuous_scraping": { - "enabled": True, + "enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True, diff --git a/health.py b/health.py index 2bdaed8..1d05f4f 100644 --- a/health.py +++ b/health.py @@ -35,7 +35,13 @@ def health_payload( "path": str(session_file), }, } - checks["accounts"] = account_checks + checks["accounts"] = { + "ok": all( + item.get("data_dir", {}).get("ok", False) + for item in account_checks.values() + ), + "items": account_checks, + } ok = all(item.get("ok", False) for item in checks.values()) return {"ok": ok, "status": "ok" if ok else "degraded", "checks": checks} diff --git a/webui/app.js b/webui/app.js index f3766dc..cb9bc72 100644 --- a/webui/app.js +++ b/webui/app.js @@ -11,6 +11,7 @@ const state = { }; const ACTIVE_ACCOUNT_STORAGE_KEY = "telegramScraper.activeAccount"; +const jobStreams = new Map(); /* ── API helper ─────────────────────────────────────── */ @@ -70,6 +71,38 @@ function confirmAction(message) { return window.confirm(message); } +function showToast(message, type = "info") { + let root = document.getElementById("toast-root"); + if (!root) { + root = document.createElement("div"); + root.id = "toast-root"; + root.className = "toast-root"; + document.body.appendChild(root); + } + const toast = document.createElement("div"); + toast.className = `toast toast-${type}`; + toast.textContent = message; + root.appendChild(toast); + window.setTimeout(() => { + toast.classList.add("toast-hide"); + window.setTimeout(() => toast.remove(), 250); + }, 3600); +} + +function downloadJson(filename, payload) { + const blob = new Blob([JSON.stringify(payload, null, 2) + "\n"], { + type: "application/json", + }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); +} + function isAccountAuthorized(auth) { const status = auth?.auth_status || auth || {}; return ( @@ -203,7 +236,15 @@ function renderAccountPanel(accountId) { container.appendChild(node); } +function closeAllJobStreams() { + for (const [jobId, stream] of jobStreams) { + stream.close(); + } + jobStreams.clear(); +} + function switchAccount(accountId) { + closeAllJobStreams(); state.activeAccount = accountId; saveActiveAccount(accountId); @@ -257,6 +298,12 @@ function renderChannels(accountId, channels) { const template = document.getElementById("channel-row-template"); tbody.innerHTML = ""; + if (!channels.length) { + const row = document.createElement("tr"); + row.innerHTML = '
No tracked channels yet. Add an ID or @username to start scraping this account.
'; + tbody.appendChild(row); + } + channels.forEach((channel) => { const node = template.content.firstElementChild.cloneNode(true); node.querySelector(".channel-name").textContent = channel.name; @@ -317,12 +364,13 @@ function renderJobs(accountId, jobs) { root.innerHTML = ""; if (!jobs.length) { - root.innerHTML = '
No jobs yet.
'; + root.innerHTML = '
No jobs yet. Start a scrape or export to see progress here.
'; return; } jobs.forEach((job) => { const node = template.content.firstElementChild.cloneNode(true); + node.dataset.jobId = job.job_id; node.querySelector(".job-title").textContent = job.title; node.querySelector(".job-status").textContent = job.status; node.querySelector(".job-time").textContent = [displayTime(job.started_at), displayTime(job.finished_at)] @@ -330,9 +378,42 @@ function renderJobs(accountId, jobs) { .join(" -> "); node.querySelector(".job-logs").textContent = job.logs || job.error || "No logs."; root.appendChild(node); + subscribeJobStream(accountId, job.job_id, job.status); }); } +function updateRenderedJob(accountId, job) { + const panel = document.getElementById(`panel-${accountId}`); + const node = panel?.querySelector(`[data-job-id="${job.job_id}"]`); + if (!node) return; + node.querySelector(".job-status").textContent = job.status; + node.querySelector(".job-time").textContent = [displayTime(job.started_at), displayTime(job.finished_at)] + .filter((item) => item !== "-") + .join(" -> "); + node.querySelector(".job-logs").textContent = job.logs || job.error || "No logs."; +} + +function subscribeJobStream(accountId, jobId, status) { + if (!["queued", "running"].includes(status) || jobStreams.has(jobId) || !window.EventSource) { + return; + } + const stream = new EventSource(`/api/jobs/${encodeURIComponent(jobId)}/events`); + jobStreams.set(jobId, stream); + stream.onmessage = (event) => { + const job = JSON.parse(event.data); + updateRenderedJob(accountId, job); + if (["completed", "failed"].includes(job.status)) { + stream.close(); + jobStreams.delete(jobId); + refreshAccount(accountId); + } + }; + stream.onerror = () => { + stream.close(); + jobStreams.delete(jobId); + }; +} + function renderContinuous(accountId, data) { const panel = document.getElementById(`panel-${accountId}`); if (!panel) return; @@ -447,9 +528,18 @@ function renderSummary(accountId, data) { const panel = document.getElementById(`panel-${accountId}`); if (!panel) return; const d = data.dashboard || data.state || {}; + const health = data.health || {}; panel.querySelector(".forwarding-count").textContent = String((d.forwarding_rules || []).length); const toggle = panel.querySelector(".scrape-media-label"); toggle.textContent = d.scrape_media ? "ON" : "OFF"; + const healthOk = health.api_credentials && health.session_ready && health.data_dir_exists; + panel.querySelector(".account-health-label").textContent = healthOk ? "Ready" : "Check"; + panel.querySelector(".account-health-label").style.color = healthOk ? "var(--ok)" : "var(--warn)"; + panel.querySelector(".account-health-detail").textContent = [ + `${health.message_count || 0} messages`, + `${health.media_count || 0} media`, + health.active_job ? `active: ${health.active_job.status}` : "idle", + ].join(" | "); } /* ── Account data loading ───────────────────────────── */ @@ -550,6 +640,16 @@ function renderSettingsAccounts() { document.getElementById("settings-dialog").close(); }); + node.querySelector(".account-export-btn").addEventListener("click", async () => { + try { + const payload = await api(`/api/accounts/${acc.id}/export`); + downloadJson(`telegram-scraper-account-${acc.id}.json`, payload); + showToast(`Exported ${acc.label || acc.id}.`, "success"); + } catch (err) { + showToast(`Failed to export account: ${err.message}`, "error"); + } + }); + node.querySelector(".account-remove-btn").addEventListener("click", async () => { if (!confirmAction(`Remove account "${acc.label || acc.id}"? All its data will be deleted.`)) return; try { @@ -561,7 +661,7 @@ function renderSettingsAccounts() { } await loadAccounts(); } catch (err) { - alert(`Failed to remove account: ${err.message}`); + showToast(`Failed to remove account: ${err.message}`, "error"); } }); @@ -677,13 +777,13 @@ async function toggleMedia(value) { async function main() { // ── Sidebar action buttons ── document.getElementById("scrape-all-btn").addEventListener("click", async () => { - try { await scrapeAll(); } catch (err) { alert("Scrape error: " + err.message); } + try { await scrapeAll(); showToast("Scrape job queued.", "success"); } catch (err) { showToast("Scrape error: " + err.message, "error"); } }); document.getElementById("export-all-btn").addEventListener("click", async () => { - try { await exportAll(); } catch (err) { alert("Export error: " + err.message); } + try { await exportAll(); showToast("Export job queued.", "success"); } catch (err) { showToast("Export error: " + err.message, "error"); } }); document.getElementById("refresh-dialogs-btn").addEventListener("click", async () => { - try { await refreshDialogs(); } catch (err) { alert("Dialog refresh error: " + err.message); } + try { await refreshDialogs(); showToast("Dialog refresh queued.", "success"); } catch (err) { showToast("Dialog refresh error: " + err.message, "error"); } }); // ── Settings dialog ── @@ -725,7 +825,27 @@ async function main() { switchAccount(accountId); } } catch (err) { - alert("Failed to add account: " + err.message); + showToast("Failed to add account: " + err.message, "error"); + } + }); + + document.getElementById("import-account-file").addEventListener("change", async (event) => { + const file = event.currentTarget.files?.[0]; + if (!file) return; + try { + const payload = JSON.parse(await file.text()); + const accountId = payload.account_id || payload.id; + if (!accountId) throw new Error("account_id is missing in import file"); + await api("/api/accounts/import", { + method: "POST", + body: JSON.stringify(payload), + }); + event.currentTarget.value = ""; + await loadAccounts(); + switchAccount(accountId); + showToast(`Imported ${accountId}.`, "success"); + } catch (err) { + showToast(`Failed to import account: ${err.message}`, "error"); } }); @@ -736,7 +856,7 @@ async function main() { try { await saveCredentials(state.activeAccount); } catch (err) { - alert("Failed to save credentials: " + err.message); + showToast("Failed to save credentials: " + err.message, "error"); } }); @@ -747,7 +867,7 @@ async function main() { if (!state.activeAccount) return; await startQrLogin(state.activeAccount); } catch (err) { - alert("QR login error: " + err.message); + showToast("QR login error: " + err.message, "error"); } finally { setBusy(event.currentTarget, false); } @@ -760,7 +880,7 @@ async function main() { try { await requestPhoneCode(state.activeAccount); } catch (err) { - alert("Phone code request error: " + err.message); + showToast("Phone code request error: " + err.message, "error"); } }); @@ -771,7 +891,7 @@ async function main() { try { await submitPhoneCode(state.activeAccount); } catch (err) { - alert("Code submit error: " + err.message); + showToast("Code submit error: " + err.message, "error"); } }); @@ -782,14 +902,14 @@ async function main() { try { await submitPassword(state.activeAccount); } catch (err) { - alert("Password error: " + err.message); + showToast("Password error: " + err.message, "error"); } }); // ── Media toggle ── document.getElementById("scrape-media-toggle").addEventListener("change", async (event) => { const checked = event.currentTarget.checked; - try { await toggleMedia(checked); } catch (err) { alert("Media toggle error: " + err.message); } + try { await toggleMedia(checked); showToast("Media setting updated.", "success"); } catch (err) { showToast("Media toggle error: " + err.message, "error"); } }); // ── Load accounts ── @@ -805,5 +925,5 @@ async function main() { main().catch((error) => { console.error(error); - alert(error.message); + showToast(error.message, "error"); }); diff --git a/webui/index.html b/webui/index.html index b979a7c..dcc7bb5 100644 --- a/webui/index.html +++ b/webui/index.html @@ -81,6 +81,10 @@ + @@ -165,6 +169,11 @@
Forwarding rules
0
+
+
Account health
+
-
+
+
@@ -300,6 +309,7 @@
+
diff --git a/webui/style.css b/webui/style.css index 2e40e14..1450ffc 100644 --- a/webui/style.css +++ b/webui/style.css @@ -638,6 +638,10 @@ tbody tr:hover { padding: 16px 24px; border-bottom: 1px solid var(--line); background: #050505; + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 16px; } .viewer-header h2 { @@ -647,6 +651,31 @@ tbody tr:hover { letter-spacing: -0.04em; } +.viewer-tools { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; + flex-wrap: wrap; +} + +.viewer-tools input[type="search"] { + min-width: min(260px, 40vw); +} + +.viewer-auto-refresh { + display: inline-flex; + align-items: center; + gap: 6px; + min-height: 34px; + color: var(--dim); + font-size: 0.82rem; +} + +.viewer-auto-refresh input { + min-width: auto; +} + .messages-list { flex: 1; overflow-y: auto; @@ -852,6 +881,54 @@ tbody tr:hover { display: none; } +.empty-state, +.viewer-empty-state { + padding: 16px; + color: var(--dim); + border: 1px solid var(--line); + border-radius: 14px; + background: #0a0a0a; +} + +.viewer-empty-state { + align-self: center; + margin-top: 16px; +} + +.toast-root { + position: fixed; + right: 18px; + bottom: 18px; + z-index: 1000; + display: grid; + gap: 8px; + width: min(360px, calc(100vw - 36px)); +} + +.toast { + padding: 12px 14px; + border: 1px solid var(--line); + border-radius: 14px; + background: #111; + color: var(--text-color); + box-shadow: 0 12px 28px rgba(0, 0, 0, 0.36); + transition: opacity 0.2s ease, transform 0.2s ease; +} + +.toast-success { + border-color: #1f5b52; +} + +.toast-error { + border-color: #7a2a2a; + color: #ffd4d4; +} + +.toast-hide { + opacity: 0; + transform: translateY(8px); +} + .api-docs { display: grid; gap: 16px; @@ -1030,6 +1107,16 @@ tbody tr:hover { background: #0f0f0f; } +.import-account-row { + display: grid; + gap: 8px; + margin-top: 10px; + padding: 12px; + border: 1px solid var(--line); + border-radius: 16px; + background: #0f0f0f; +} + /* ── Active Account Card ─────────────────────────────── */ #active-account-name { diff --git a/webui/viewer.html b/webui/viewer.html index 96eae71..0a5147f 100644 --- a/webui/viewer.html +++ b/webui/viewer.html @@ -27,6 +27,14 @@

Select a channel

Reading messages from the local SQLite database.

+
+ + + +
diff --git a/webui/viewer.js b/webui/viewer.js index 48431d2..0a504f6 100644 --- a/webui/viewer.js +++ b/webui/viewer.js @@ -8,7 +8,9 @@ const viewerState = { oldestMessageId: null, newestMessageId: null, userId: null, + search: "", loading: false, + autoRefreshTimer: null, sentinelObserver: null, }; @@ -21,6 +23,24 @@ async function api(path) { return data; } +function showToast(message, type = "info") { + let root = document.getElementById("toast-root"); + if (!root) { + root = document.createElement("div"); + root.id = "toast-root"; + root.className = "toast-root"; + document.body.appendChild(root); + } + const toast = document.createElement("div"); + toast.className = `toast toast-${type}`; + toast.textContent = message; + root.appendChild(toast); + window.setTimeout(() => { + toast.classList.add("toast-hide"); + window.setTimeout(() => toast.remove(), 250); + }, 3600); +} + function formatDateHeader(dateStr) { if (!dateStr) return ""; const d = new Date(dateStr.replace(" ", "T")); @@ -136,10 +156,13 @@ function renderChannelList() { } function messageEndpoint(channelId, before = "") { + const search = viewerState.search + ? `&search=${encodeURIComponent(viewerState.search)}` + : ""; if (viewerState.accountId) { - return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`; + return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`; } - return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`; + return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`; } function channelsEndpoint() { @@ -172,6 +195,9 @@ function renderAccountSelector() { async function switchViewerAccount(accountId) { viewerState.accountId = accountId || null; + viewerState.search = ""; + const searchInput = document.getElementById("viewer-search"); + if (searchInput) searchInput.value = ""; viewerState.accounts.forEach((acc) => { if (acc.id === accountId) { const opts = document.getElementById("viewer-account-select")?.options; @@ -323,7 +349,7 @@ function renderMessages(payload, append = false) { document.getElementById("viewer-title").textContent = channel?.name || payload.channel_id; document.getElementById("viewer-subtitle").textContent = channel - ? `${channel.message_count} messages, latest: ${channel.last_date || "-"}` + ? `${payload.messages.length} shown${viewerState.search ? ` for "${viewerState.search}"` : ""}, ${channel.message_count} total, latest: ${channel.last_date || "-"}` : "No local database found for this channel yet."; if (!append) { @@ -371,6 +397,15 @@ function renderMessages(payload, append = false) { viewerState.oldestMessageId = payload.messages[0].message_id; viewerState.newestMessageId = payload.messages[payload.messages.length - 1].message_id; + } else if (!append) { + viewerState.oldestMessageId = null; + viewerState.newestMessageId = null; + const empty = document.createElement("div"); + empty.className = "viewer-empty-state"; + empty.textContent = viewerState.search + ? `No messages found for "${viewerState.search}".` + : "No saved messages in this channel yet."; + root.appendChild(empty); } if (!append) { @@ -407,6 +442,40 @@ async function loadChannel(channelId, append = false) { } } +async function refreshCurrentChannel() { + if (!viewerState.channelId || viewerState.loading) return; + viewerState.oldestMessageId = null; + viewerState.newestMessageId = null; + await loadChannel(viewerState.channelId); +} + +function setupViewerTools() { + const searchInput = document.getElementById("viewer-search"); + const refreshBtn = document.getElementById("viewer-refresh-btn"); + const autoRefresh = document.getElementById("viewer-auto-refresh"); + let searchTimer = null; + + if (searchInput) { + searchInput.addEventListener("input", () => { + clearTimeout(searchTimer); + searchTimer = setTimeout(() => { + viewerState.search = searchInput.value.trim(); + refreshCurrentChannel(); + }, 250); + }); + } + + if (refreshBtn) { + refreshBtn.addEventListener("click", () => refreshCurrentChannel()); + } + + viewerState.autoRefreshTimer = window.setInterval(() => { + if (!autoRefresh || autoRefresh.checked) { + refreshCurrentChannel(); + } + }, 30000); +} + function setupInfiniteScroll() { const sentinel = document.getElementById("scroll-sentinel"); if (!sentinel) return; @@ -456,6 +525,7 @@ async function main() { } setupInfiniteScroll(); + setupViewerTools(); const toggle = document.getElementById("sidebar-toggle"); const sidebar = document.querySelector(".viewer-sidebar"); @@ -473,5 +543,5 @@ async function main() { main().catch((error) => { console.error(error); - alert(error.message); + showToast(error.message, "error"); }); diff --git a/webui_server.py b/webui_server.py index 031e78d..b8a8ef6 100644 --- a/webui_server.py +++ b/webui_server.py @@ -194,6 +194,7 @@ def load_messages( channel_id: str, limit: int = 120, before_message_id: Optional[int] = None, + search: Optional[str] = None, ) -> List[Dict[str, Any]]: db_path = channel_db_path(account_id, channel_id) if not db_path.exists(): @@ -207,9 +208,15 @@ def load_messages( "FROM messages " ) params: List[Any] = [] + where: List[str] = [] if before_message_id is not None: - query += "WHERE message_id < ? " + where.append("message_id < ?") params.append(before_message_id) + if search: + where.append("message LIKE ?") + params.append(f"%{search}%") + if where: + query += "WHERE " + " AND ".join(where) + " " query += "ORDER BY message_id DESC LIMIT ?" params.append(limit) rows = conn.execute(query, params).fetchall() @@ -323,16 +330,29 @@ class JobRunner: def create_job(self, job_type: str, title: str, payload: Dict[str, Any]) -> Job: if self._shutdown_flag: raise RuntimeError("Server is shutting down, cannot create new jobs") - job_id = f"job-{int(time.time() * 1000)}" account_id = payload.get("account_id") - job = Job( - job_id=job_id, - job_type=job_type, - title=title, - payload=payload, - account_id=account_id, - ) with self.lock: + if account_id: + for job_id in self.job_order: + existing = self.jobs[job_id] + if ( + existing.account_id == account_id + and existing.status in {"queued", "running"} + ): + existing.logs = ( + existing.logs + + "\n" + + f"[{datetime.now().strftime('%H:%M:%S')}] Reused existing active job for this account." + ).strip() + return existing + job_id = f"job-{int(time.time() * 1000)}" + job = Job( + job_id=job_id, + job_type=job_type, + title=title, + payload=payload, + account_id=account_id, + ) self.jobs[job_id] = job self.job_order.insert(0, job_id) self.job_order = self.job_order[:20] @@ -346,6 +366,15 @@ class JobRunner: result = [j for j in result if j.get("account_id") == account_id] return result + def active_jobs_by_account(self) -> Dict[str, Dict[str, Any]]: + with self.lock: + active = {} + for job_id in self.job_order: + job = self.jobs[job_id] + if job.account_id and job.status in {"queued", "running"}: + active[job.account_id] = job.to_dict() + return active + def get_job(self, job_id: str) -> Optional[Dict[str, Any]]: with self.lock: job = self.jobs.get(job_id) @@ -811,6 +840,20 @@ class PerAccountContinuousScrapeManager: def _run_loop(self) -> None: while not self.stop_event.is_set(): + # Check auth — don't iterate if account isn't authorized + auth_info = auth_status_for(self.account_id) + if auth_info.get("status") not in ("ready", "authorized"): + self._log( + f"Account not authorized (status={auth_info.get('status')}), " + "skipping iteration", + "warn", + ) + sleep_seconds = max(5, 60) + interrupted = self.stop_event.wait(timeout=sleep_seconds) + if interrupted: + break + continue + channels = self._resolve_channels() cfg = self.snapshot()["config"] interval_minutes = cfg.get("interval_minutes", 1) @@ -904,10 +947,10 @@ class ContinuousScrapeOrchestrator: mgr = self._get_or_create(account_id) return mgr.update(enabled, interval_minutes, channels, run_all_tracked) - def add_account(self, account_id: str) -> None: + def add_account(self, account_id: str, auto_start: bool = True) -> None: acc_state = load_account(DATA_DIR, account_id) cfg = acc_state.get("continuous_scraping", {}) - if cfg.get("enabled", True) and START_CONTINUOUS: + if auto_start and cfg.get("enabled", False) and START_CONTINUOUS: self.start_account(account_id) def remove_account(self, account_id: str) -> None: @@ -924,7 +967,7 @@ class ContinuousScrapeOrchestrator: for account_id in list_accounts(DATA_DIR): acc_state = load_account(DATA_DIR, account_id) cfg = acc_state.get("continuous_scraping", {}) - if cfg.get("enabled", True): + if cfg.get("enabled", False) and START_CONTINUOUS: self.start_account(account_id) @@ -973,6 +1016,27 @@ def auth_status_for(account_id: Optional[str] = None) -> Dict[str, Any]: return status +def account_health_summary(account_id: str, job_runner: Optional[JobRunner] = None) -> Dict[str, Any]: + acc_state = load_account(DATA_DIR, account_id) + acc_dir = account_data_dir(DATA_DIR, account_id) + session_file = Path(account_session_path(SESSION_DIR, account_id)) + channels = list_channels_snapshot(account_id) + active_job = (job_runner.active_jobs_by_account().get(account_id) if job_runner else None) + last_scrape = next((item.get("last_date") for item in channels if item.get("last_date")), None) + return { + "account_id": account_id, + "label": acc_state.get("label", ""), + "data_dir_exists": acc_dir.exists(), + "session_ready": session_file.exists(), + "api_credentials": bool(acc_state.get("api_id") and acc_state.get("api_hash")), + "channel_count": len(channels), + "message_count": sum(int(item.get("message_count") or 0) for item in channels), + "media_count": sum(int(item.get("media_count") or 0) for item in channels), + "last_scrape": last_scrape, + "active_job": active_job, + } + + def auth_status() -> Dict[str, Any]: return auth_status_for(account_id=None) @@ -1166,6 +1230,12 @@ def openapi_payload() -> Dict[str, Any]: "in": "query", "schema": {"type": "integer"}, }, + { + "name": "search", + "in": "query", + "schema": {"type": "string"}, + "description": "Full-text search in message text", + }, ], "responses": json_response, } @@ -1229,6 +1299,31 @@ def openapi_payload() -> Dict[str, Any]: }, } }, + "/api/jobs/{job_id}/events": { + "get": { + "summary": "Job SSE event stream", + "description": "Server-Sent Events stream of job status updates. Closes when job completes or fails.", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + } + ], + "responses": { + "200": { + "description": "SSE stream of job status objects", + "content": { + "text/event-stream": { + "schema": {"type": "string"} + } + }, + }, + "404": {"description": "Job not found"}, + }, + } + }, "/api/jobs/scrape": { "post": { "summary": "Scrape one channel or all tracked channels", @@ -1318,6 +1413,23 @@ def openapi_payload() -> Dict[str, Any]: "responses": {**json_response, **error_response}, }, }, + "/api/accounts/import": { + "post": { + "summary": "Import account from exported JSON", + "requestBody": json_body( + { + "account_id": {"type": "string", "example": "work"}, + "state": { + "type": "object", + "description": "Exported account state object", + "example": {"label": "Work", "api_id": 123456, "api_hash": "abc", "channels": {}, "continuous_scraping": {"enabled": False}}, + }, + }, + ["account_id", "state"], + ), + "responses": {**json_response, **error_response}, + } + }, "/api/accounts/{id}": { "get": { "summary": "Account dashboard", @@ -1344,6 +1456,34 @@ def openapi_payload() -> Dict[str, Any]: "responses": json_response, }, }, + "/api/accounts/{id}/export": { + "get": { + "summary": "Export account settings as JSON", + "parameters": [ + { + "name": "id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + } + ], + "responses": json_response, + } + }, + "/api/accounts/{id}/health": { + "get": { + "summary": "Account health summary", + "parameters": [ + { + "name": "id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + } + ], + "responses": json_response, + } + }, "/api/accounts/{id}/auth": { "get": { "summary": "Account auth snapshot", @@ -1452,6 +1592,12 @@ def openapi_payload() -> Dict[str, Any]: "in": "query", "schema": {"type": "integer"}, }, + { + "name": "search", + "in": "query", + "schema": {"type": "string"}, + "description": "Full-text search in message text", + }, ], "responses": json_response, } @@ -1634,6 +1780,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): ) if path == "/api/jobs": return self.send_json(self.app.job_runner.recent_jobs()) + if path.startswith("/api/jobs/") and path.endswith("/events"): + job_id = path.split("/")[-2] + return self.stream_job_events(job_id) if path.startswith("/api/jobs/"): job_id = path.rsplit("/", 1)[-1] job = self.app.job_runner.get_job(job_id) @@ -1703,6 +1852,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): "auth": auth_info, "status": auth_info.get("status", "unknown"), "continuous_running": cs_info.get("status", {}).get("running", False), + "health": account_health_summary(account_id, self.app.job_runner), }) return self.send_json({"accounts": accounts}) @@ -1719,6 +1869,15 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self._handle_get_account_dashboard(account_id) if sub == ["auth"]: return self.send_json(self.app.auth_manager.auth_state(account_id)) + if sub == ["health"]: + return self.send_json(account_health_summary(account_id, self.app.job_runner)) + if sub == ["export"]: + payload = { + "version": 1, + "account_id": account_id, + "state": load_account(DATA_DIR, account_id), + } + return self.send_json(payload) if sub == ["channels"]: return self._handle_get_account_channels(account_id) if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages": @@ -1744,9 +1903,37 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): "jobs": self.app.job_runner.recent_jobs(account_id=account_id), "continuous": cs_info, "scrape_media": bool(acc_state.get("scrape_media", True)), + "health": account_health_summary(account_id, self.app.job_runner), } return self.send_json(payload) + def stream_job_events(self, job_id: str) -> None: + if not self.app.job_runner.get_job(job_id): + return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found") + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "keep-alive") + self.end_headers() + + last_payload = None + deadline = time.time() + 60 * 30 + while time.time() < deadline: + job = self.app.job_runner.get_job(job_id) + if not job: + break + payload = json.dumps(job, ensure_ascii=False) + if payload != last_payload: + try: + self.wfile.write(f"data: {payload}\n\n".encode("utf-8")) + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError, OSError): + break + last_payload = payload + if job.get("status") in {"completed", "failed"}: + break + time.sleep(1) + def _handle_get_account_channels(self, account_id: str) -> None: return self.send_json(list_channels_snapshot(account_id)) @@ -1756,10 +1943,12 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): limit = max(1, min(int(query.get("limit", ["120"])[0]), 300)) before = query.get("before") before_message_id = int(before[0]) if before else None + search = (query.get("search") or query.get("q") or [""])[0].strip() payload = { "channel_id": channel_id, "messages": load_messages( account_id, channel_id, limit=limit, before_message_id=before_message_id + , search=search or None ), "channel": next( ( @@ -1997,6 +2186,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): # ── /api/accounts/* POST routes ───────────────────────────────── if path == "/api/accounts": return self._handle_post_accounts_create(body) + if path == "/api/accounts/import": + return self._handle_post_accounts_import(body) if path.startswith("/api/accounts/"): return self._handle_post_account(path, body) @@ -2034,14 +2225,52 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): "scrape_media": True, "forwarding_rules": [], "continuous_scraping": { - "enabled": True, + "enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True, }, } store.save(init_state) - self.app.continuous_orchestrator.add_account(account_id) + self.app.continuous_orchestrator.add_account(account_id, auto_start=False) + return self.send_json({"ok": True, "account_id": account_id}) + + def _handle_post_accounts_import(self, body: Dict[str, Any]) -> None: + account_id = str(body.get("account_id") or body.get("id") or "").strip() + state = body.get("state") + if not account_id: + return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id is required") + if not account_id.replace("-", "").replace("_", "").isalnum(): + return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id must be alphanumeric (dashes and underscores allowed)") + if account_exists(DATA_DIR, account_id): + return self.send_error_json(HTTPStatus.BAD_REQUEST, f"Account '{account_id}' already exists") + if not isinstance(state, dict): + return self.send_error_json(HTTPStatus.BAD_REQUEST, "state object is required") + + imported_state = { + "label": str(body.get("label") or state.get("label") or account_id).strip(), + "api_id": state.get("api_id"), + "api_hash": state.get("api_hash"), + "channels": state.get("channels") if isinstance(state.get("channels"), dict) else {}, + "channel_names": state.get("channel_names") if isinstance(state.get("channel_names"), dict) else {}, + "scrape_media": bool(state.get("scrape_media", True)), + "forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [], + "continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else { + "enabled": False, + "interval_minutes": 1, + "channels": [], + "run_all_tracked": True, + }, + } + + def mutate_global(global_state: Dict[str, Any]) -> None: + accounts = global_state.setdefault("accounts", []) + if account_id not in accounts: + accounts.append(account_id) + + get_global_store(DATA_DIR).update(mutate_global) + get_account_store(DATA_DIR, account_id).save(imported_state) + self.app.continuous_orchestrator.add_account(account_id, auto_start=False) return self.send_json({"ok": True, "account_id": account_id}) def _handle_post_account(self, path: str, body: Dict[str, Any]) -> None: @@ -2300,9 +2529,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if acc_dir.exists(): import shutil shutil.rmtree(str(acc_dir), ignore_errors=True) - if session_file.exists(): + for session_sidecar in ( + session_file, + Path(str(session_file) + "-wal"), + Path(str(session_file) + "-shm"), + session_file.with_suffix(session_file.suffix + "-journal"), + ): + if not session_sidecar.exists(): + continue try: - session_file.unlink() + session_sidecar.unlink() except OSError: pass