Harden account import/continuous behavior and finish multi-account UI polish

This commit is contained in:
2026-06-27 21:26:18 +02:00
parent 59cdad220a
commit 881e7ddfca
8 changed files with 572 additions and 35 deletions
+1 -1
View File
@@ -25,7 +25,7 @@ ACCOUNT_DEFAULTS: Dict[str, Any] = {
"scrape_media": True, "scrape_media": True,
"forwarding_rules": [], "forwarding_rules": [],
"continuous_scraping": { "continuous_scraping": {
"enabled": True, "enabled": False,
"interval_minutes": 1, "interval_minutes": 1,
"channels": [], "channels": [],
"run_all_tracked": True, "run_all_tracked": True,
+7 -1
View File
@@ -35,7 +35,13 @@ def health_payload(
"path": str(session_file), "path": str(session_file),
}, },
} }
checks["accounts"] = account_checks checks["accounts"] = {
"ok": all(
item.get("data_dir", {}).get("ok", False)
for item in account_checks.values()
),
"items": account_checks,
}
ok = all(item.get("ok", False) for item in checks.values()) ok = all(item.get("ok", False) for item in checks.values())
return {"ok": ok, "status": "ok" if ok else "degraded", "checks": checks} return {"ok": ok, "status": "ok" if ok else "degraded", "checks": checks}
+133 -13
View File
@@ -11,6 +11,7 @@ const state = {
}; };
const ACTIVE_ACCOUNT_STORAGE_KEY = "telegramScraper.activeAccount"; const ACTIVE_ACCOUNT_STORAGE_KEY = "telegramScraper.activeAccount";
const jobStreams = new Map();
/* ── API helper ─────────────────────────────────────── */ /* ── API helper ─────────────────────────────────────── */
@@ -70,6 +71,38 @@ function confirmAction(message) {
return window.confirm(message); return window.confirm(message);
} }
function showToast(message, type = "info") {
let root = document.getElementById("toast-root");
if (!root) {
root = document.createElement("div");
root.id = "toast-root";
root.className = "toast-root";
document.body.appendChild(root);
}
const toast = document.createElement("div");
toast.className = `toast toast-${type}`;
toast.textContent = message;
root.appendChild(toast);
window.setTimeout(() => {
toast.classList.add("toast-hide");
window.setTimeout(() => toast.remove(), 250);
}, 3600);
}
function downloadJson(filename, payload) {
const blob = new Blob([JSON.stringify(payload, null, 2) + "\n"], {
type: "application/json",
});
const url = URL.createObjectURL(blob);
const link = document.createElement("a");
link.href = url;
link.download = filename;
document.body.appendChild(link);
link.click();
link.remove();
URL.revokeObjectURL(url);
}
function isAccountAuthorized(auth) { function isAccountAuthorized(auth) {
const status = auth?.auth_status || auth || {}; const status = auth?.auth_status || auth || {};
return ( return (
@@ -203,7 +236,15 @@ function renderAccountPanel(accountId) {
container.appendChild(node); container.appendChild(node);
} }
function closeAllJobStreams() {
for (const [jobId, stream] of jobStreams) {
stream.close();
}
jobStreams.clear();
}
function switchAccount(accountId) { function switchAccount(accountId) {
closeAllJobStreams();
state.activeAccount = accountId; state.activeAccount = accountId;
saveActiveAccount(accountId); saveActiveAccount(accountId);
@@ -257,6 +298,12 @@ function renderChannels(accountId, channels) {
const template = document.getElementById("channel-row-template"); const template = document.getElementById("channel-row-template");
tbody.innerHTML = ""; tbody.innerHTML = "";
if (!channels.length) {
const row = document.createElement("tr");
row.innerHTML = '<td colspan="5"><div class="empty-state">No tracked channels yet. Add an ID or @username to start scraping this account.</div></td>';
tbody.appendChild(row);
}
channels.forEach((channel) => { channels.forEach((channel) => {
const node = template.content.firstElementChild.cloneNode(true); const node = template.content.firstElementChild.cloneNode(true);
node.querySelector(".channel-name").textContent = channel.name; node.querySelector(".channel-name").textContent = channel.name;
@@ -317,12 +364,13 @@ function renderJobs(accountId, jobs) {
root.innerHTML = ""; root.innerHTML = "";
if (!jobs.length) { if (!jobs.length) {
root.innerHTML = '<div class="muted">No jobs yet.</div>'; root.innerHTML = '<div class="empty-state">No jobs yet. Start a scrape or export to see progress here.</div>';
return; return;
} }
jobs.forEach((job) => { jobs.forEach((job) => {
const node = template.content.firstElementChild.cloneNode(true); const node = template.content.firstElementChild.cloneNode(true);
node.dataset.jobId = job.job_id;
node.querySelector(".job-title").textContent = job.title; node.querySelector(".job-title").textContent = job.title;
node.querySelector(".job-status").textContent = job.status; node.querySelector(".job-status").textContent = job.status;
node.querySelector(".job-time").textContent = [displayTime(job.started_at), displayTime(job.finished_at)] node.querySelector(".job-time").textContent = [displayTime(job.started_at), displayTime(job.finished_at)]
@@ -330,9 +378,42 @@ function renderJobs(accountId, jobs) {
.join(" -> "); .join(" -> ");
node.querySelector(".job-logs").textContent = job.logs || job.error || "No logs."; node.querySelector(".job-logs").textContent = job.logs || job.error || "No logs.";
root.appendChild(node); root.appendChild(node);
subscribeJobStream(accountId, job.job_id, job.status);
}); });
} }
function updateRenderedJob(accountId, job) {
const panel = document.getElementById(`panel-${accountId}`);
const node = panel?.querySelector(`[data-job-id="${job.job_id}"]`);
if (!node) return;
node.querySelector(".job-status").textContent = job.status;
node.querySelector(".job-time").textContent = [displayTime(job.started_at), displayTime(job.finished_at)]
.filter((item) => item !== "-")
.join(" -> ");
node.querySelector(".job-logs").textContent = job.logs || job.error || "No logs.";
}
function subscribeJobStream(accountId, jobId, status) {
if (!["queued", "running"].includes(status) || jobStreams.has(jobId) || !window.EventSource) {
return;
}
const stream = new EventSource(`/api/jobs/${encodeURIComponent(jobId)}/events`);
jobStreams.set(jobId, stream);
stream.onmessage = (event) => {
const job = JSON.parse(event.data);
updateRenderedJob(accountId, job);
if (["completed", "failed"].includes(job.status)) {
stream.close();
jobStreams.delete(jobId);
refreshAccount(accountId);
}
};
stream.onerror = () => {
stream.close();
jobStreams.delete(jobId);
};
}
function renderContinuous(accountId, data) { function renderContinuous(accountId, data) {
const panel = document.getElementById(`panel-${accountId}`); const panel = document.getElementById(`panel-${accountId}`);
if (!panel) return; if (!panel) return;
@@ -447,9 +528,18 @@ function renderSummary(accountId, data) {
const panel = document.getElementById(`panel-${accountId}`); const panel = document.getElementById(`panel-${accountId}`);
if (!panel) return; if (!panel) return;
const d = data.dashboard || data.state || {}; const d = data.dashboard || data.state || {};
const health = data.health || {};
panel.querySelector(".forwarding-count").textContent = String((d.forwarding_rules || []).length); panel.querySelector(".forwarding-count").textContent = String((d.forwarding_rules || []).length);
const toggle = panel.querySelector(".scrape-media-label"); const toggle = panel.querySelector(".scrape-media-label");
toggle.textContent = d.scrape_media ? "ON" : "OFF"; toggle.textContent = d.scrape_media ? "ON" : "OFF";
const healthOk = health.api_credentials && health.session_ready && health.data_dir_exists;
panel.querySelector(".account-health-label").textContent = healthOk ? "Ready" : "Check";
panel.querySelector(".account-health-label").style.color = healthOk ? "var(--ok)" : "var(--warn)";
panel.querySelector(".account-health-detail").textContent = [
`${health.message_count || 0} messages`,
`${health.media_count || 0} media`,
health.active_job ? `active: ${health.active_job.status}` : "idle",
].join(" | ");
} }
/* ── Account data loading ───────────────────────────── */ /* ── Account data loading ───────────────────────────── */
@@ -550,6 +640,16 @@ function renderSettingsAccounts() {
document.getElementById("settings-dialog").close(); document.getElementById("settings-dialog").close();
}); });
node.querySelector(".account-export-btn").addEventListener("click", async () => {
try {
const payload = await api(`/api/accounts/${acc.id}/export`);
downloadJson(`telegram-scraper-account-${acc.id}.json`, payload);
showToast(`Exported ${acc.label || acc.id}.`, "success");
} catch (err) {
showToast(`Failed to export account: ${err.message}`, "error");
}
});
node.querySelector(".account-remove-btn").addEventListener("click", async () => { node.querySelector(".account-remove-btn").addEventListener("click", async () => {
if (!confirmAction(`Remove account "${acc.label || acc.id}"? All its data will be deleted.`)) return; if (!confirmAction(`Remove account "${acc.label || acc.id}"? All its data will be deleted.`)) return;
try { try {
@@ -561,7 +661,7 @@ function renderSettingsAccounts() {
} }
await loadAccounts(); await loadAccounts();
} catch (err) { } catch (err) {
alert(`Failed to remove account: ${err.message}`); showToast(`Failed to remove account: ${err.message}`, "error");
} }
}); });
@@ -677,13 +777,13 @@ async function toggleMedia(value) {
async function main() { async function main() {
// ── Sidebar action buttons ── // ── Sidebar action buttons ──
document.getElementById("scrape-all-btn").addEventListener("click", async () => { document.getElementById("scrape-all-btn").addEventListener("click", async () => {
try { await scrapeAll(); } catch (err) { alert("Scrape error: " + err.message); } try { await scrapeAll(); showToast("Scrape job queued.", "success"); } catch (err) { showToast("Scrape error: " + err.message, "error"); }
}); });
document.getElementById("export-all-btn").addEventListener("click", async () => { document.getElementById("export-all-btn").addEventListener("click", async () => {
try { await exportAll(); } catch (err) { alert("Export error: " + err.message); } try { await exportAll(); showToast("Export job queued.", "success"); } catch (err) { showToast("Export error: " + err.message, "error"); }
}); });
document.getElementById("refresh-dialogs-btn").addEventListener("click", async () => { document.getElementById("refresh-dialogs-btn").addEventListener("click", async () => {
try { await refreshDialogs(); } catch (err) { alert("Dialog refresh error: " + err.message); } try { await refreshDialogs(); showToast("Dialog refresh queued.", "success"); } catch (err) { showToast("Dialog refresh error: " + err.message, "error"); }
}); });
// ── Settings dialog ── // ── Settings dialog ──
@@ -725,7 +825,27 @@ async function main() {
switchAccount(accountId); switchAccount(accountId);
} }
} catch (err) { } catch (err) {
alert("Failed to add account: " + err.message); showToast("Failed to add account: " + err.message, "error");
}
});
document.getElementById("import-account-file").addEventListener("change", async (event) => {
const file = event.currentTarget.files?.[0];
if (!file) return;
try {
const payload = JSON.parse(await file.text());
const accountId = payload.account_id || payload.id;
if (!accountId) throw new Error("account_id is missing in import file");
await api("/api/accounts/import", {
method: "POST",
body: JSON.stringify(payload),
});
event.currentTarget.value = "";
await loadAccounts();
switchAccount(accountId);
showToast(`Imported ${accountId}.`, "success");
} catch (err) {
showToast(`Failed to import account: ${err.message}`, "error");
} }
}); });
@@ -736,7 +856,7 @@ async function main() {
try { try {
await saveCredentials(state.activeAccount); await saveCredentials(state.activeAccount);
} catch (err) { } catch (err) {
alert("Failed to save credentials: " + err.message); showToast("Failed to save credentials: " + err.message, "error");
} }
}); });
@@ -747,7 +867,7 @@ async function main() {
if (!state.activeAccount) return; if (!state.activeAccount) return;
await startQrLogin(state.activeAccount); await startQrLogin(state.activeAccount);
} catch (err) { } catch (err) {
alert("QR login error: " + err.message); showToast("QR login error: " + err.message, "error");
} finally { } finally {
setBusy(event.currentTarget, false); setBusy(event.currentTarget, false);
} }
@@ -760,7 +880,7 @@ async function main() {
try { try {
await requestPhoneCode(state.activeAccount); await requestPhoneCode(state.activeAccount);
} catch (err) { } catch (err) {
alert("Phone code request error: " + err.message); showToast("Phone code request error: " + err.message, "error");
} }
}); });
@@ -771,7 +891,7 @@ async function main() {
try { try {
await submitPhoneCode(state.activeAccount); await submitPhoneCode(state.activeAccount);
} catch (err) { } catch (err) {
alert("Code submit error: " + err.message); showToast("Code submit error: " + err.message, "error");
} }
}); });
@@ -782,14 +902,14 @@ async function main() {
try { try {
await submitPassword(state.activeAccount); await submitPassword(state.activeAccount);
} catch (err) { } catch (err) {
alert("Password error: " + err.message); showToast("Password error: " + err.message, "error");
} }
}); });
// ── Media toggle ── // ── Media toggle ──
document.getElementById("scrape-media-toggle").addEventListener("change", async (event) => { document.getElementById("scrape-media-toggle").addEventListener("change", async (event) => {
const checked = event.currentTarget.checked; const checked = event.currentTarget.checked;
try { await toggleMedia(checked); } catch (err) { alert("Media toggle error: " + err.message); } try { await toggleMedia(checked); showToast("Media setting updated.", "success"); } catch (err) { showToast("Media toggle error: " + err.message, "error"); }
}); });
// ── Load accounts ── // ── Load accounts ──
@@ -805,5 +925,5 @@ async function main() {
main().catch((error) => { main().catch((error) => {
console.error(error); console.error(error);
alert(error.message); showToast(error.message, "error");
}); });
+10
View File
@@ -81,6 +81,10 @@
<input id="add-account-api-hash" name="api_hash" placeholder="API Hash" /> <input id="add-account-api-hash" name="api_hash" placeholder="API Hash" />
<button class="button primary" type="submit">Add Account</button> <button class="button primary" type="submit">Add Account</button>
</form> </form>
<label class="import-account-row">
<span class="muted small">Import account settings JSON</span>
<input id="import-account-file" type="file" accept="application/json,.json" />
</label>
</section> </section>
<!-- ── Account Auth ── (shown per account selected in UI) --> <!-- ── Account Auth ── (shown per account selected in UI) -->
@@ -165,6 +169,11 @@
<div class="section-title">Forwarding rules</div> <div class="section-title">Forwarding rules</div>
<div class="stat-value forwarding-count">0</div> <div class="stat-value forwarding-count">0</div>
</div> </div>
<div class="panel stat-panel">
<div class="section-title">Account health</div>
<div class="stat-value stat-compact account-health-label">-</div>
<div class="muted small account-health-detail"></div>
</div>
</section> </section>
<section class="panel"> <section class="panel">
@@ -300,6 +309,7 @@
</div> </div>
<div class="account-list-actions"> <div class="account-list-actions">
<button class="button button-small account-select-btn">Select</button> <button class="button button-small account-select-btn">Select</button>
<button class="button button-small account-export-btn">Export</button>
<button class="button button-small danger account-remove-btn">Remove</button> <button class="button button-small danger account-remove-btn">Remove</button>
</div> </div>
</div> </div>
+87
View File
@@ -638,6 +638,10 @@ tbody tr:hover {
padding: 16px 24px; padding: 16px 24px;
border-bottom: 1px solid var(--line); border-bottom: 1px solid var(--line);
background: #050505; background: #050505;
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
} }
.viewer-header h2 { .viewer-header h2 {
@@ -647,6 +651,31 @@ tbody tr:hover {
letter-spacing: -0.04em; letter-spacing: -0.04em;
} }
.viewer-tools {
display: flex;
align-items: center;
justify-content: flex-end;
gap: 8px;
flex-wrap: wrap;
}
.viewer-tools input[type="search"] {
min-width: min(260px, 40vw);
}
.viewer-auto-refresh {
display: inline-flex;
align-items: center;
gap: 6px;
min-height: 34px;
color: var(--dim);
font-size: 0.82rem;
}
.viewer-auto-refresh input {
min-width: auto;
}
.messages-list { .messages-list {
flex: 1; flex: 1;
overflow-y: auto; overflow-y: auto;
@@ -852,6 +881,54 @@ tbody tr:hover {
display: none; display: none;
} }
.empty-state,
.viewer-empty-state {
padding: 16px;
color: var(--dim);
border: 1px solid var(--line);
border-radius: 14px;
background: #0a0a0a;
}
.viewer-empty-state {
align-self: center;
margin-top: 16px;
}
.toast-root {
position: fixed;
right: 18px;
bottom: 18px;
z-index: 1000;
display: grid;
gap: 8px;
width: min(360px, calc(100vw - 36px));
}
.toast {
padding: 12px 14px;
border: 1px solid var(--line);
border-radius: 14px;
background: #111;
color: var(--text-color);
box-shadow: 0 12px 28px rgba(0, 0, 0, 0.36);
transition: opacity 0.2s ease, transform 0.2s ease;
}
.toast-success {
border-color: #1f5b52;
}
.toast-error {
border-color: #7a2a2a;
color: #ffd4d4;
}
.toast-hide {
opacity: 0;
transform: translateY(8px);
}
.api-docs { .api-docs {
display: grid; display: grid;
gap: 16px; gap: 16px;
@@ -1030,6 +1107,16 @@ tbody tr:hover {
background: #0f0f0f; background: #0f0f0f;
} }
.import-account-row {
display: grid;
gap: 8px;
margin-top: 10px;
padding: 12px;
border: 1px solid var(--line);
border-radius: 16px;
background: #0f0f0f;
}
/* ── Active Account Card ─────────────────────────────── */ /* ── Active Account Card ─────────────────────────────── */
#active-account-name { #active-account-name {
+8
View File
@@ -27,6 +27,14 @@
<h2 id="viewer-title">Select a channel</h2> <h2 id="viewer-title">Select a channel</h2>
<p id="viewer-subtitle" class="muted">Reading messages from the local SQLite database.</p> <p id="viewer-subtitle" class="muted">Reading messages from the local SQLite database.</p>
</div> </div>
<div class="viewer-tools">
<input id="viewer-search" type="search" placeholder="Search messages" />
<label class="viewer-auto-refresh">
<input id="viewer-auto-refresh" type="checkbox" checked />
<span>Auto-refresh</span>
</label>
<button id="viewer-refresh-btn" class="button button-small" type="button">Refresh</button>
</div>
</header> </header>
<section id="messages-list" class="messages-list"> <section id="messages-list" class="messages-list">
+74 -4
View File
@@ -8,7 +8,9 @@ const viewerState = {
oldestMessageId: null, oldestMessageId: null,
newestMessageId: null, newestMessageId: null,
userId: null, userId: null,
search: "",
loading: false, loading: false,
autoRefreshTimer: null,
sentinelObserver: null, sentinelObserver: null,
}; };
@@ -21,6 +23,24 @@ async function api(path) {
return data; return data;
} }
function showToast(message, type = "info") {
let root = document.getElementById("toast-root");
if (!root) {
root = document.createElement("div");
root.id = "toast-root";
root.className = "toast-root";
document.body.appendChild(root);
}
const toast = document.createElement("div");
toast.className = `toast toast-${type}`;
toast.textContent = message;
root.appendChild(toast);
window.setTimeout(() => {
toast.classList.add("toast-hide");
window.setTimeout(() => toast.remove(), 250);
}, 3600);
}
function formatDateHeader(dateStr) { function formatDateHeader(dateStr) {
if (!dateStr) return ""; if (!dateStr) return "";
const d = new Date(dateStr.replace(" ", "T")); const d = new Date(dateStr.replace(" ", "T"));
@@ -136,10 +156,13 @@ function renderChannelList() {
} }
function messageEndpoint(channelId, before = "") { function messageEndpoint(channelId, before = "") {
const search = viewerState.search
? `&search=${encodeURIComponent(viewerState.search)}`
: "";
if (viewerState.accountId) { if (viewerState.accountId) {
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`; return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
} }
return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`; return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
} }
function channelsEndpoint() { function channelsEndpoint() {
@@ -172,6 +195,9 @@ function renderAccountSelector() {
async function switchViewerAccount(accountId) { async function switchViewerAccount(accountId) {
viewerState.accountId = accountId || null; viewerState.accountId = accountId || null;
viewerState.search = "";
const searchInput = document.getElementById("viewer-search");
if (searchInput) searchInput.value = "";
viewerState.accounts.forEach((acc) => { viewerState.accounts.forEach((acc) => {
if (acc.id === accountId) { if (acc.id === accountId) {
const opts = document.getElementById("viewer-account-select")?.options; const opts = document.getElementById("viewer-account-select")?.options;
@@ -323,7 +349,7 @@ function renderMessages(payload, append = false) {
document.getElementById("viewer-title").textContent = document.getElementById("viewer-title").textContent =
channel?.name || payload.channel_id; channel?.name || payload.channel_id;
document.getElementById("viewer-subtitle").textContent = channel document.getElementById("viewer-subtitle").textContent = channel
? `${channel.message_count} messages, latest: ${channel.last_date || "-"}` ? `${payload.messages.length} shown${viewerState.search ? ` for "${viewerState.search}"` : ""}, ${channel.message_count} total, latest: ${channel.last_date || "-"}`
: "No local database found for this channel yet."; : "No local database found for this channel yet.";
if (!append) { if (!append) {
@@ -371,6 +397,15 @@ function renderMessages(payload, append = false) {
viewerState.oldestMessageId = payload.messages[0].message_id; viewerState.oldestMessageId = payload.messages[0].message_id;
viewerState.newestMessageId = viewerState.newestMessageId =
payload.messages[payload.messages.length - 1].message_id; payload.messages[payload.messages.length - 1].message_id;
} else if (!append) {
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
const empty = document.createElement("div");
empty.className = "viewer-empty-state";
empty.textContent = viewerState.search
? `No messages found for "${viewerState.search}".`
: "No saved messages in this channel yet.";
root.appendChild(empty);
} }
if (!append) { if (!append) {
@@ -407,6 +442,40 @@ async function loadChannel(channelId, append = false) {
} }
} }
async function refreshCurrentChannel() {
if (!viewerState.channelId || viewerState.loading) return;
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
await loadChannel(viewerState.channelId);
}
function setupViewerTools() {
const searchInput = document.getElementById("viewer-search");
const refreshBtn = document.getElementById("viewer-refresh-btn");
const autoRefresh = document.getElementById("viewer-auto-refresh");
let searchTimer = null;
if (searchInput) {
searchInput.addEventListener("input", () => {
clearTimeout(searchTimer);
searchTimer = setTimeout(() => {
viewerState.search = searchInput.value.trim();
refreshCurrentChannel();
}, 250);
});
}
if (refreshBtn) {
refreshBtn.addEventListener("click", () => refreshCurrentChannel());
}
viewerState.autoRefreshTimer = window.setInterval(() => {
if (!autoRefresh || autoRefresh.checked) {
refreshCurrentChannel();
}
}, 30000);
}
function setupInfiniteScroll() { function setupInfiniteScroll() {
const sentinel = document.getElementById("scroll-sentinel"); const sentinel = document.getElementById("scroll-sentinel");
if (!sentinel) return; if (!sentinel) return;
@@ -456,6 +525,7 @@ async function main() {
} }
setupInfiniteScroll(); setupInfiniteScroll();
setupViewerTools();
const toggle = document.getElementById("sidebar-toggle"); const toggle = document.getElementById("sidebar-toggle");
const sidebar = document.querySelector(".viewer-sidebar"); const sidebar = document.querySelector(".viewer-sidebar");
@@ -473,5 +543,5 @@ async function main() {
main().catch((error) => { main().catch((error) => {
console.error(error); console.error(error);
alert(error.message); showToast(error.message, "error");
}); });
+252 -16
View File
@@ -194,6 +194,7 @@ def load_messages(
channel_id: str, channel_id: str,
limit: int = 120, limit: int = 120,
before_message_id: Optional[int] = None, before_message_id: Optional[int] = None,
search: Optional[str] = None,
) -> List[Dict[str, Any]]: ) -> List[Dict[str, Any]]:
db_path = channel_db_path(account_id, channel_id) db_path = channel_db_path(account_id, channel_id)
if not db_path.exists(): if not db_path.exists():
@@ -207,9 +208,15 @@ def load_messages(
"FROM messages " "FROM messages "
) )
params: List[Any] = [] params: List[Any] = []
where: List[str] = []
if before_message_id is not None: if before_message_id is not None:
query += "WHERE message_id < ? " where.append("message_id < ?")
params.append(before_message_id) params.append(before_message_id)
if search:
where.append("message LIKE ?")
params.append(f"%{search}%")
if where:
query += "WHERE " + " AND ".join(where) + " "
query += "ORDER BY message_id DESC LIMIT ?" query += "ORDER BY message_id DESC LIMIT ?"
params.append(limit) params.append(limit)
rows = conn.execute(query, params).fetchall() rows = conn.execute(query, params).fetchall()
@@ -323,16 +330,29 @@ class JobRunner:
def create_job(self, job_type: str, title: str, payload: Dict[str, Any]) -> Job: def create_job(self, job_type: str, title: str, payload: Dict[str, Any]) -> Job:
if self._shutdown_flag: if self._shutdown_flag:
raise RuntimeError("Server is shutting down, cannot create new jobs") raise RuntimeError("Server is shutting down, cannot create new jobs")
job_id = f"job-{int(time.time() * 1000)}"
account_id = payload.get("account_id") account_id = payload.get("account_id")
job = Job(
job_id=job_id,
job_type=job_type,
title=title,
payload=payload,
account_id=account_id,
)
with self.lock: with self.lock:
if account_id:
for job_id in self.job_order:
existing = self.jobs[job_id]
if (
existing.account_id == account_id
and existing.status in {"queued", "running"}
):
existing.logs = (
existing.logs
+ "\n"
+ f"[{datetime.now().strftime('%H:%M:%S')}] Reused existing active job for this account."
).strip()
return existing
job_id = f"job-{int(time.time() * 1000)}"
job = Job(
job_id=job_id,
job_type=job_type,
title=title,
payload=payload,
account_id=account_id,
)
self.jobs[job_id] = job self.jobs[job_id] = job
self.job_order.insert(0, job_id) self.job_order.insert(0, job_id)
self.job_order = self.job_order[:20] self.job_order = self.job_order[:20]
@@ -346,6 +366,15 @@ class JobRunner:
result = [j for j in result if j.get("account_id") == account_id] result = [j for j in result if j.get("account_id") == account_id]
return result return result
def active_jobs_by_account(self) -> Dict[str, Dict[str, Any]]:
with self.lock:
active = {}
for job_id in self.job_order:
job = self.jobs[job_id]
if job.account_id and job.status in {"queued", "running"}:
active[job.account_id] = job.to_dict()
return active
def get_job(self, job_id: str) -> Optional[Dict[str, Any]]: def get_job(self, job_id: str) -> Optional[Dict[str, Any]]:
with self.lock: with self.lock:
job = self.jobs.get(job_id) job = self.jobs.get(job_id)
@@ -811,6 +840,20 @@ class PerAccountContinuousScrapeManager:
def _run_loop(self) -> None: def _run_loop(self) -> None:
while not self.stop_event.is_set(): while not self.stop_event.is_set():
# Check auth — don't iterate if account isn't authorized
auth_info = auth_status_for(self.account_id)
if auth_info.get("status") not in ("ready", "authorized"):
self._log(
f"Account not authorized (status={auth_info.get('status')}), "
"skipping iteration",
"warn",
)
sleep_seconds = max(5, 60)
interrupted = self.stop_event.wait(timeout=sleep_seconds)
if interrupted:
break
continue
channels = self._resolve_channels() channels = self._resolve_channels()
cfg = self.snapshot()["config"] cfg = self.snapshot()["config"]
interval_minutes = cfg.get("interval_minutes", 1) interval_minutes = cfg.get("interval_minutes", 1)
@@ -904,10 +947,10 @@ class ContinuousScrapeOrchestrator:
mgr = self._get_or_create(account_id) mgr = self._get_or_create(account_id)
return mgr.update(enabled, interval_minutes, channels, run_all_tracked) return mgr.update(enabled, interval_minutes, channels, run_all_tracked)
def add_account(self, account_id: str) -> None: def add_account(self, account_id: str, auto_start: bool = True) -> None:
acc_state = load_account(DATA_DIR, account_id) acc_state = load_account(DATA_DIR, account_id)
cfg = acc_state.get("continuous_scraping", {}) cfg = acc_state.get("continuous_scraping", {})
if cfg.get("enabled", True) and START_CONTINUOUS: if auto_start and cfg.get("enabled", False) and START_CONTINUOUS:
self.start_account(account_id) self.start_account(account_id)
def remove_account(self, account_id: str) -> None: def remove_account(self, account_id: str) -> None:
@@ -924,7 +967,7 @@ class ContinuousScrapeOrchestrator:
for account_id in list_accounts(DATA_DIR): for account_id in list_accounts(DATA_DIR):
acc_state = load_account(DATA_DIR, account_id) acc_state = load_account(DATA_DIR, account_id)
cfg = acc_state.get("continuous_scraping", {}) cfg = acc_state.get("continuous_scraping", {})
if cfg.get("enabled", True): if cfg.get("enabled", False) and START_CONTINUOUS:
self.start_account(account_id) self.start_account(account_id)
@@ -973,6 +1016,27 @@ def auth_status_for(account_id: Optional[str] = None) -> Dict[str, Any]:
return status return status
def account_health_summary(account_id: str, job_runner: Optional[JobRunner] = None) -> Dict[str, Any]:
acc_state = load_account(DATA_DIR, account_id)
acc_dir = account_data_dir(DATA_DIR, account_id)
session_file = Path(account_session_path(SESSION_DIR, account_id))
channels = list_channels_snapshot(account_id)
active_job = (job_runner.active_jobs_by_account().get(account_id) if job_runner else None)
last_scrape = next((item.get("last_date") for item in channels if item.get("last_date")), None)
return {
"account_id": account_id,
"label": acc_state.get("label", ""),
"data_dir_exists": acc_dir.exists(),
"session_ready": session_file.exists(),
"api_credentials": bool(acc_state.get("api_id") and acc_state.get("api_hash")),
"channel_count": len(channels),
"message_count": sum(int(item.get("message_count") or 0) for item in channels),
"media_count": sum(int(item.get("media_count") or 0) for item in channels),
"last_scrape": last_scrape,
"active_job": active_job,
}
def auth_status() -> Dict[str, Any]: def auth_status() -> Dict[str, Any]:
return auth_status_for(account_id=None) return auth_status_for(account_id=None)
@@ -1166,6 +1230,12 @@ def openapi_payload() -> Dict[str, Any]:
"in": "query", "in": "query",
"schema": {"type": "integer"}, "schema": {"type": "integer"},
}, },
{
"name": "search",
"in": "query",
"schema": {"type": "string"},
"description": "Full-text search in message text",
},
], ],
"responses": json_response, "responses": json_response,
} }
@@ -1229,6 +1299,31 @@ def openapi_payload() -> Dict[str, Any]:
}, },
} }
}, },
"/api/jobs/{job_id}/events": {
"get": {
"summary": "Job SSE event stream",
"description": "Server-Sent Events stream of job status updates. Closes when job completes or fails.",
"parameters": [
{
"name": "job_id",
"in": "path",
"required": True,
"schema": {"type": "string"},
}
],
"responses": {
"200": {
"description": "SSE stream of job status objects",
"content": {
"text/event-stream": {
"schema": {"type": "string"}
}
},
},
"404": {"description": "Job not found"},
},
}
},
"/api/jobs/scrape": { "/api/jobs/scrape": {
"post": { "post": {
"summary": "Scrape one channel or all tracked channels", "summary": "Scrape one channel or all tracked channels",
@@ -1318,6 +1413,23 @@ def openapi_payload() -> Dict[str, Any]:
"responses": {**json_response, **error_response}, "responses": {**json_response, **error_response},
}, },
}, },
"/api/accounts/import": {
"post": {
"summary": "Import account from exported JSON",
"requestBody": json_body(
{
"account_id": {"type": "string", "example": "work"},
"state": {
"type": "object",
"description": "Exported account state object",
"example": {"label": "Work", "api_id": 123456, "api_hash": "abc", "channels": {}, "continuous_scraping": {"enabled": False}},
},
},
["account_id", "state"],
),
"responses": {**json_response, **error_response},
}
},
"/api/accounts/{id}": { "/api/accounts/{id}": {
"get": { "get": {
"summary": "Account dashboard", "summary": "Account dashboard",
@@ -1344,6 +1456,34 @@ def openapi_payload() -> Dict[str, Any]:
"responses": json_response, "responses": json_response,
}, },
}, },
"/api/accounts/{id}/export": {
"get": {
"summary": "Export account settings as JSON",
"parameters": [
{
"name": "id",
"in": "path",
"required": True,
"schema": {"type": "string"},
}
],
"responses": json_response,
}
},
"/api/accounts/{id}/health": {
"get": {
"summary": "Account health summary",
"parameters": [
{
"name": "id",
"in": "path",
"required": True,
"schema": {"type": "string"},
}
],
"responses": json_response,
}
},
"/api/accounts/{id}/auth": { "/api/accounts/{id}/auth": {
"get": { "get": {
"summary": "Account auth snapshot", "summary": "Account auth snapshot",
@@ -1452,6 +1592,12 @@ def openapi_payload() -> Dict[str, Any]:
"in": "query", "in": "query",
"schema": {"type": "integer"}, "schema": {"type": "integer"},
}, },
{
"name": "search",
"in": "query",
"schema": {"type": "string"},
"description": "Full-text search in message text",
},
], ],
"responses": json_response, "responses": json_response,
} }
@@ -1634,6 +1780,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
) )
if path == "/api/jobs": if path == "/api/jobs":
return self.send_json(self.app.job_runner.recent_jobs()) return self.send_json(self.app.job_runner.recent_jobs())
if path.startswith("/api/jobs/") and path.endswith("/events"):
job_id = path.split("/")[-2]
return self.stream_job_events(job_id)
if path.startswith("/api/jobs/"): if path.startswith("/api/jobs/"):
job_id = path.rsplit("/", 1)[-1] job_id = path.rsplit("/", 1)[-1]
job = self.app.job_runner.get_job(job_id) job = self.app.job_runner.get_job(job_id)
@@ -1703,6 +1852,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
"auth": auth_info, "auth": auth_info,
"status": auth_info.get("status", "unknown"), "status": auth_info.get("status", "unknown"),
"continuous_running": cs_info.get("status", {}).get("running", False), "continuous_running": cs_info.get("status", {}).get("running", False),
"health": account_health_summary(account_id, self.app.job_runner),
}) })
return self.send_json({"accounts": accounts}) return self.send_json({"accounts": accounts})
@@ -1719,6 +1869,15 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self._handle_get_account_dashboard(account_id) return self._handle_get_account_dashboard(account_id)
if sub == ["auth"]: if sub == ["auth"]:
return self.send_json(self.app.auth_manager.auth_state(account_id)) return self.send_json(self.app.auth_manager.auth_state(account_id))
if sub == ["health"]:
return self.send_json(account_health_summary(account_id, self.app.job_runner))
if sub == ["export"]:
payload = {
"version": 1,
"account_id": account_id,
"state": load_account(DATA_DIR, account_id),
}
return self.send_json(payload)
if sub == ["channels"]: if sub == ["channels"]:
return self._handle_get_account_channels(account_id) return self._handle_get_account_channels(account_id)
if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages": if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages":
@@ -1744,9 +1903,37 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
"jobs": self.app.job_runner.recent_jobs(account_id=account_id), "jobs": self.app.job_runner.recent_jobs(account_id=account_id),
"continuous": cs_info, "continuous": cs_info,
"scrape_media": bool(acc_state.get("scrape_media", True)), "scrape_media": bool(acc_state.get("scrape_media", True)),
"health": account_health_summary(account_id, self.app.job_runner),
} }
return self.send_json(payload) return self.send_json(payload)
def stream_job_events(self, job_id: str) -> None:
if not self.app.job_runner.get_job(job_id):
return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found")
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "keep-alive")
self.end_headers()
last_payload = None
deadline = time.time() + 60 * 30
while time.time() < deadline:
job = self.app.job_runner.get_job(job_id)
if not job:
break
payload = json.dumps(job, ensure_ascii=False)
if payload != last_payload:
try:
self.wfile.write(f"data: {payload}\n\n".encode("utf-8"))
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, OSError):
break
last_payload = payload
if job.get("status") in {"completed", "failed"}:
break
time.sleep(1)
def _handle_get_account_channels(self, account_id: str) -> None: def _handle_get_account_channels(self, account_id: str) -> None:
return self.send_json(list_channels_snapshot(account_id)) return self.send_json(list_channels_snapshot(account_id))
@@ -1756,10 +1943,12 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300)) limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
before = query.get("before") before = query.get("before")
before_message_id = int(before[0]) if before else None before_message_id = int(before[0]) if before else None
search = (query.get("search") or query.get("q") or [""])[0].strip()
payload = { payload = {
"channel_id": channel_id, "channel_id": channel_id,
"messages": load_messages( "messages": load_messages(
account_id, channel_id, limit=limit, before_message_id=before_message_id account_id, channel_id, limit=limit, before_message_id=before_message_id
, search=search or None
), ),
"channel": next( "channel": next(
( (
@@ -1997,6 +2186,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
# ── /api/accounts/* POST routes ───────────────────────────────── # ── /api/accounts/* POST routes ─────────────────────────────────
if path == "/api/accounts": if path == "/api/accounts":
return self._handle_post_accounts_create(body) return self._handle_post_accounts_create(body)
if path == "/api/accounts/import":
return self._handle_post_accounts_import(body)
if path.startswith("/api/accounts/"): if path.startswith("/api/accounts/"):
return self._handle_post_account(path, body) return self._handle_post_account(path, body)
@@ -2034,14 +2225,52 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
"scrape_media": True, "scrape_media": True,
"forwarding_rules": [], "forwarding_rules": [],
"continuous_scraping": { "continuous_scraping": {
"enabled": True, "enabled": False,
"interval_minutes": 1, "interval_minutes": 1,
"channels": [], "channels": [],
"run_all_tracked": True, "run_all_tracked": True,
}, },
} }
store.save(init_state) store.save(init_state)
self.app.continuous_orchestrator.add_account(account_id) self.app.continuous_orchestrator.add_account(account_id, auto_start=False)
return self.send_json({"ok": True, "account_id": account_id})
def _handle_post_accounts_import(self, body: Dict[str, Any]) -> None:
account_id = str(body.get("account_id") or body.get("id") or "").strip()
state = body.get("state")
if not account_id:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id is required")
if not account_id.replace("-", "").replace("_", "").isalnum():
return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id must be alphanumeric (dashes and underscores allowed)")
if account_exists(DATA_DIR, account_id):
return self.send_error_json(HTTPStatus.BAD_REQUEST, f"Account '{account_id}' already exists")
if not isinstance(state, dict):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "state object is required")
imported_state = {
"label": str(body.get("label") or state.get("label") or account_id).strip(),
"api_id": state.get("api_id"),
"api_hash": state.get("api_hash"),
"channels": state.get("channels") if isinstance(state.get("channels"), dict) else {},
"channel_names": state.get("channel_names") if isinstance(state.get("channel_names"), dict) else {},
"scrape_media": bool(state.get("scrape_media", True)),
"forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [],
"continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else {
"enabled": False,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
},
}
def mutate_global(global_state: Dict[str, Any]) -> None:
accounts = global_state.setdefault("accounts", [])
if account_id not in accounts:
accounts.append(account_id)
get_global_store(DATA_DIR).update(mutate_global)
get_account_store(DATA_DIR, account_id).save(imported_state)
self.app.continuous_orchestrator.add_account(account_id, auto_start=False)
return self.send_json({"ok": True, "account_id": account_id}) return self.send_json({"ok": True, "account_id": account_id})
def _handle_post_account(self, path: str, body: Dict[str, Any]) -> None: def _handle_post_account(self, path: str, body: Dict[str, Any]) -> None:
@@ -2300,9 +2529,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if acc_dir.exists(): if acc_dir.exists():
import shutil import shutil
shutil.rmtree(str(acc_dir), ignore_errors=True) shutil.rmtree(str(acc_dir), ignore_errors=True)
if session_file.exists(): for session_sidecar in (
session_file,
Path(str(session_file) + "-wal"),
Path(str(session_file) + "-shm"),
session_file.with_suffix(session_file.suffix + "-journal"),
):
if not session_sidecar.exists():
continue
try: try:
session_file.unlink() session_sidecar.unlink()
except OSError: except OSError:
pass pass