Add web auth and continuous scraping logs
This commit is contained in:
@@ -1 +0,0 @@
|
||||
ko_fi: unnohwn
|
||||
+2
-1
@@ -3,7 +3,8 @@ data/
|
||||
.python-version
|
||||
uv.lock
|
||||
.uv-cache/
|
||||
|
||||
session/
|
||||
session.session
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
+1
-3
@@ -12,6 +12,4 @@ services:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- session:/app/session
|
||||
volumes:
|
||||
session:
|
||||
- ./session:/app/session
|
||||
|
||||
+121
-1
@@ -1,5 +1,7 @@
|
||||
const state = {
|
||||
dashboard: null,
|
||||
auth: null,
|
||||
continuous: null,
|
||||
};
|
||||
|
||||
async function api(path, options = {}) {
|
||||
@@ -29,6 +31,30 @@ function renderAuth(auth) {
|
||||
document.getElementById("auth-details").textContent = auth.details || "";
|
||||
}
|
||||
|
||||
function toggleHidden(id, hidden) {
|
||||
document.getElementById(id).classList.toggle("hidden", hidden);
|
||||
}
|
||||
|
||||
function renderAuthPanel(authData) {
|
||||
state.auth = authData;
|
||||
renderAuth(authData.auth_status);
|
||||
|
||||
const apiId = authData.saved_credentials?.api_id ?? "";
|
||||
document.getElementById("api-id-input").value = apiId || "";
|
||||
document.getElementById("api-hash-input").placeholder = authData.saved_credentials?.api_hash_present
|
||||
? "API Hash сохранён"
|
||||
: "API Hash";
|
||||
|
||||
const showQr = Boolean(authData.qr_image);
|
||||
toggleHidden("qr-wrap", !showQr);
|
||||
if (showQr) {
|
||||
document.getElementById("qr-image").src = authData.qr_image;
|
||||
}
|
||||
|
||||
toggleHidden("code-form", authData.phase !== "code_required");
|
||||
toggleHidden("password-form", authData.phase !== "password_required");
|
||||
}
|
||||
|
||||
function renderSummary(data) {
|
||||
document.getElementById("channel-count").textContent = String(data.state.channel_count);
|
||||
document.getElementById("forwarding-count").textContent = String(data.state.forwarding_rules.length);
|
||||
@@ -102,13 +128,32 @@ function renderJobs(jobs) {
|
||||
});
|
||||
}
|
||||
|
||||
function renderContinuous(data) {
|
||||
state.continuous = data;
|
||||
const config = data.config || {};
|
||||
const status = data.status || {};
|
||||
|
||||
document.getElementById("continuous-enabled").checked = Boolean(config.enabled);
|
||||
document.getElementById("continuous-interval").value = config.interval_minutes || 1;
|
||||
document.getElementById("continuous-all").checked = Boolean(config.run_all_tracked);
|
||||
document.getElementById("continuous-channels").value = (config.channels || []).join(", ");
|
||||
|
||||
document.getElementById("continuous-status").textContent = status.running ? "running" : "stopped";
|
||||
document.getElementById("continuous-last-iteration").textContent = status.last_iteration_at || "—";
|
||||
document.getElementById("continuous-last-error").textContent = status.last_error || "—";
|
||||
document.getElementById("continuous-logs").textContent = (status.logs || []).join("\n") || "Логов пока нет.";
|
||||
}
|
||||
|
||||
async function refreshDashboard() {
|
||||
const data = await api("/api/dashboard");
|
||||
const authData = await api("/api/auth");
|
||||
const continuousData = await api("/api/continuous");
|
||||
state.dashboard = data;
|
||||
renderAuth(data.auth);
|
||||
renderAuthPanel(authData);
|
||||
renderSummary(data);
|
||||
renderChannels(data.channels);
|
||||
renderJobs(data.jobs);
|
||||
renderContinuous(continuousData);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
@@ -167,6 +212,81 @@ async function main() {
|
||||
await refreshDashboard();
|
||||
});
|
||||
|
||||
document.getElementById("credentials-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const apiId = document.getElementById("api-id-input").value.trim();
|
||||
const apiHash = document.getElementById("api-hash-input").value.trim();
|
||||
await api("/api/auth/credentials", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ api_id: apiId, api_hash: apiHash }),
|
||||
});
|
||||
await refreshDashboard();
|
||||
});
|
||||
|
||||
document.getElementById("start-qr-btn").addEventListener("click", async (event) => {
|
||||
setBusy(event.currentTarget, true);
|
||||
try {
|
||||
await api("/api/auth/qr/start", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
await refreshDashboard();
|
||||
} finally {
|
||||
setBusy(event.currentTarget, false);
|
||||
}
|
||||
});
|
||||
|
||||
document.getElementById("phone-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const phone = document.getElementById("phone-input").value.trim();
|
||||
await api("/api/auth/phone/request", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ phone }),
|
||||
});
|
||||
await refreshDashboard();
|
||||
});
|
||||
|
||||
document.getElementById("code-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const code = document.getElementById("code-input").value.trim();
|
||||
await api("/api/auth/phone/submit", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ code }),
|
||||
});
|
||||
event.currentTarget.reset();
|
||||
await refreshDashboard();
|
||||
});
|
||||
|
||||
document.getElementById("password-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const password = document.getElementById("password-input").value.trim();
|
||||
await api("/api/auth/password", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ password }),
|
||||
});
|
||||
event.currentTarget.reset();
|
||||
await refreshDashboard();
|
||||
});
|
||||
|
||||
document.getElementById("continuous-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const enabled = document.getElementById("continuous-enabled").checked;
|
||||
const intervalMinutes = document.getElementById("continuous-interval").value;
|
||||
const runAllTracked = document.getElementById("continuous-all").checked;
|
||||
const channelsRaw = document.getElementById("continuous-channels").value.trim();
|
||||
const channels = channelsRaw ? channelsRaw.split(",").map((item) => item.trim()).filter(Boolean) : [];
|
||||
await api("/api/continuous", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
enabled,
|
||||
interval_minutes: intervalMinutes,
|
||||
run_all_tracked: runAllTracked,
|
||||
channels,
|
||||
}),
|
||||
});
|
||||
await refreshDashboard();
|
||||
});
|
||||
|
||||
await refreshDashboard();
|
||||
window.setInterval(refreshDashboard, 5000);
|
||||
}
|
||||
|
||||
@@ -26,6 +26,39 @@
|
||||
<p id="auth-details" class="muted small"></p>
|
||||
</section>
|
||||
|
||||
<section class="status-card">
|
||||
<div class="section-title">Telegram Login</div>
|
||||
<form id="credentials-form" class="stack-form">
|
||||
<input id="api-id-input" name="api_id" placeholder="API ID" />
|
||||
<input id="api-hash-input" name="api_hash" placeholder="API Hash" />
|
||||
<button class="button" type="submit">Сохранить креды</button>
|
||||
</form>
|
||||
|
||||
<div class="auth-actions">
|
||||
<button class="button primary" id="start-qr-btn">Логин по QR</button>
|
||||
</div>
|
||||
|
||||
<div id="qr-wrap" class="qr-wrap hidden">
|
||||
<img id="qr-image" alt="Telegram QR login" />
|
||||
<p class="muted small">Открой Telegram -> Settings -> Devices -> Scan QR</p>
|
||||
</div>
|
||||
|
||||
<form id="phone-form" class="stack-form">
|
||||
<input id="phone-input" name="phone" placeholder="+1234567890" />
|
||||
<button class="button" type="submit">Отправить код</button>
|
||||
</form>
|
||||
|
||||
<form id="code-form" class="stack-form hidden">
|
||||
<input id="code-input" name="code" placeholder="Код из Telegram" />
|
||||
<button class="button" type="submit">Подтвердить код</button>
|
||||
</form>
|
||||
|
||||
<form id="password-form" class="stack-form hidden">
|
||||
<input id="password-input" name="password" type="password" placeholder="2FA password" />
|
||||
<button class="button" type="submit">Подтвердить пароль</button>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="status-card">
|
||||
<div class="section-title">Быстрые действия</div>
|
||||
<button class="button primary" id="scrape-all-btn">Скрапить все каналы</button>
|
||||
@@ -92,6 +125,43 @@
|
||||
</div>
|
||||
<div id="jobs-list" class="jobs-list"></div>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<div class="panel-header">
|
||||
<div>
|
||||
<h2>Continuous Scraping</h2>
|
||||
<p class="muted">Периодически перескрапливает каналы и пишет лог прямо в панель.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form id="continuous-form" class="continuous-form">
|
||||
<label class="toggle-row">
|
||||
<span>Включить</span>
|
||||
<input id="continuous-enabled" type="checkbox" />
|
||||
</label>
|
||||
<label class="stack-form">
|
||||
<span class="muted small">Интервал, минут</span>
|
||||
<input id="continuous-interval" type="number" min="1" value="1" />
|
||||
</label>
|
||||
<label class="toggle-row">
|
||||
<span>Все отслеживаемые каналы</span>
|
||||
<input id="continuous-all" type="checkbox" checked />
|
||||
</label>
|
||||
<label class="stack-form">
|
||||
<span class="muted small">Если выключить опцию выше, сюда можно вписать ID каналов через запятую</span>
|
||||
<input id="continuous-channels" placeholder="-100123, 5926277437" />
|
||||
</label>
|
||||
<button class="button primary" type="submit">Сохранить continuous scraping</button>
|
||||
</form>
|
||||
|
||||
<div class="continuous-meta">
|
||||
<div class="muted small">Статус: <span id="continuous-status">—</span></div>
|
||||
<div class="muted small">Последний цикл: <span id="continuous-last-iteration">—</span></div>
|
||||
<div class="muted small">Последняя ошибка: <span id="continuous-last-error">—</span></div>
|
||||
</div>
|
||||
|
||||
<pre id="continuous-logs" class="job-logs"></pre>
|
||||
</section>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -186,6 +186,38 @@ button {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.stack-form {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
.auth-actions {
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
.qr-wrap {
|
||||
margin-top: 14px;
|
||||
padding: 12px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: #0d141c;
|
||||
}
|
||||
|
||||
.qr-wrap img {
|
||||
display: block;
|
||||
width: 100%;
|
||||
max-width: 240px;
|
||||
aspect-ratio: 1;
|
||||
margin: 0 auto 10px;
|
||||
border-radius: 8px;
|
||||
background: white;
|
||||
}
|
||||
|
||||
.hidden {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
input {
|
||||
min-width: 160px;
|
||||
padding: 10px 12px;
|
||||
@@ -257,6 +289,17 @@ tbody tr:hover {
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
.continuous-form {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.continuous-meta {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.viewer-shell {
|
||||
grid-template-columns: 320px minmax(0, 1fr);
|
||||
}
|
||||
|
||||
+402
-3
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
@@ -16,7 +17,12 @@ from datetime import datetime, timezone
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import qrcode
|
||||
import qrcode.image.svg
|
||||
from telethon import TelegramClient
|
||||
from telethon.errors import SessionPasswordNeededError
|
||||
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
@@ -25,6 +31,8 @@ WEBUI_DIR = BASE_DIR / "webui"
|
||||
STATE_FILE = DATA_DIR / "state.json"
|
||||
DEFAULT_HOST = os.environ.get("TELEGRAM_SCRAPER_HOST", "0.0.0.0")
|
||||
DEFAULT_PORT = int(os.environ.get("TELEGRAM_SCRAPER_PORT", "8080"))
|
||||
SESSION_DIR = BASE_DIR / "session"
|
||||
SESSION_DIR.mkdir(exist_ok=True)
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
@@ -284,6 +292,320 @@ class JobRunner:
|
||||
job.finished_at = utc_now_iso()
|
||||
|
||||
|
||||
class TelegramAuthManager:
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.Lock()
|
||||
self.loop = asyncio.new_event_loop()
|
||||
self.thread = threading.Thread(target=self._run_loop, daemon=True)
|
||||
self.thread.start()
|
||||
self.client: Optional[TelegramClient] = None
|
||||
self.qr_login = None
|
||||
self.qr_wait_task = None
|
||||
self.phone = None
|
||||
self.phone_code_hash = None
|
||||
self.state: Dict[str, Any] = {
|
||||
"phase": "idle",
|
||||
"status": "unknown",
|
||||
"details": "",
|
||||
"qr_url": None,
|
||||
"qr_image": None,
|
||||
"phone": None,
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
|
||||
def _run_loop(self) -> None:
|
||||
asyncio.set_event_loop(self.loop)
|
||||
self.loop.run_forever()
|
||||
|
||||
def _run(self, coro):
|
||||
future = asyncio.run_coroutine_threadsafe(coro, self.loop)
|
||||
return future.result()
|
||||
|
||||
def _set_state(self, **updates: Any) -> None:
|
||||
with self.lock:
|
||||
self.state.update(updates)
|
||||
self.state["updated_at"] = utc_now_iso()
|
||||
|
||||
def snapshot(self) -> Dict[str, Any]:
|
||||
auth = auth_status()
|
||||
with self.lock:
|
||||
snapshot = dict(self.state)
|
||||
snapshot["saved_credentials"] = {
|
||||
"api_id": load_state().get("api_id"),
|
||||
"api_hash_present": bool(load_state().get("api_hash")),
|
||||
}
|
||||
snapshot["auth_status"] = auth
|
||||
return snapshot
|
||||
|
||||
async def _get_client(self) -> TelegramClient:
|
||||
state = load_state()
|
||||
api_id = state.get("api_id")
|
||||
api_hash = state.get("api_hash")
|
||||
if not api_id or not api_hash:
|
||||
raise RuntimeError("Save api_id and api_hash first.")
|
||||
if self.client is None:
|
||||
self.client = TelegramClient(str(SESSION_DIR / "session"), api_id, api_hash)
|
||||
if not self.client.is_connected():
|
||||
await self.client.connect()
|
||||
return self.client
|
||||
|
||||
async def _is_authorized(self) -> bool:
|
||||
client = await self._get_client()
|
||||
return await client.is_user_authorized()
|
||||
|
||||
def save_credentials(self, api_id: int, api_hash: str) -> Dict[str, Any]:
|
||||
state = load_state()
|
||||
state["api_id"] = int(api_id)
|
||||
state["api_hash"] = api_hash.strip()
|
||||
save_state(state)
|
||||
self._set_state(
|
||||
phase="credentials_saved",
|
||||
status="ready_for_auth",
|
||||
details="Credentials saved. You can login with QR or phone code.",
|
||||
)
|
||||
return self.snapshot()
|
||||
|
||||
def _make_qr_image(self, qr_url: str) -> str:
|
||||
qr = qrcode.QRCode(border=1, box_size=8)
|
||||
qr.add_data(qr_url)
|
||||
qr.make(fit=True)
|
||||
image = qr.make_image(image_factory=qrcode.image.svg.SvgPathImage)
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer)
|
||||
encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
return f"data:image/svg+xml;base64,{encoded}"
|
||||
|
||||
async def _wait_for_qr_login(self) -> None:
|
||||
try:
|
||||
await self.qr_login.wait()
|
||||
self._set_state(
|
||||
phase="authorized",
|
||||
status="authorized",
|
||||
details="Telegram session is authorized.",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
phone=None,
|
||||
)
|
||||
except SessionPasswordNeededError:
|
||||
self._set_state(
|
||||
phase="password_required",
|
||||
status="password_required",
|
||||
details="Two-factor authentication is enabled. Enter your Telegram password.",
|
||||
)
|
||||
except Exception as exc:
|
||||
self._set_state(
|
||||
phase="error",
|
||||
status="error",
|
||||
details=f"QR login failed: {exc}",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
)
|
||||
|
||||
async def _start_qr_login(self) -> Dict[str, Any]:
|
||||
client = await self._get_client()
|
||||
if await client.is_user_authorized():
|
||||
self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.")
|
||||
return self.snapshot()
|
||||
self.qr_login = await client.qr_login()
|
||||
qr_url = self.qr_login.url
|
||||
self._set_state(
|
||||
phase="qr_waiting",
|
||||
status="qr_waiting",
|
||||
details="Scan the QR code in Telegram: Settings -> Devices -> Scan QR.",
|
||||
qr_url=qr_url,
|
||||
qr_image=self._make_qr_image(qr_url),
|
||||
)
|
||||
self.qr_wait_task = self.loop.create_task(self._wait_for_qr_login())
|
||||
return self.snapshot()
|
||||
|
||||
def start_qr_login(self) -> Dict[str, Any]:
|
||||
return self._run(self._start_qr_login())
|
||||
|
||||
async def _request_phone_code(self, phone: str) -> Dict[str, Any]:
|
||||
client = await self._get_client()
|
||||
if await client.is_user_authorized():
|
||||
self._set_state(phase="authorized", status="authorized", details="Telegram session is already authorized.")
|
||||
return self.snapshot()
|
||||
sent = await client.send_code_request(phone)
|
||||
self.phone = phone
|
||||
self.phone_code_hash = sent.phone_code_hash
|
||||
self._set_state(
|
||||
phase="code_required",
|
||||
status="code_required",
|
||||
details=f"Code sent to {phone}. Enter it below.",
|
||||
phone=phone,
|
||||
)
|
||||
return self.snapshot()
|
||||
|
||||
def request_phone_code(self, phone: str) -> Dict[str, Any]:
|
||||
return self._run(self._request_phone_code(phone))
|
||||
|
||||
async def _submit_phone_code(self, code: str) -> Dict[str, Any]:
|
||||
client = await self._get_client()
|
||||
if not self.phone or not self.phone_code_hash:
|
||||
raise RuntimeError("Request a phone code first.")
|
||||
try:
|
||||
await client.sign_in(phone=self.phone, code=code, phone_code_hash=self.phone_code_hash)
|
||||
self._set_state(
|
||||
phase="authorized",
|
||||
status="authorized",
|
||||
details="Telegram session is authorized.",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
)
|
||||
except SessionPasswordNeededError:
|
||||
self._set_state(
|
||||
phase="password_required",
|
||||
status="password_required",
|
||||
details="Two-factor authentication is enabled. Enter your Telegram password.",
|
||||
)
|
||||
return self.snapshot()
|
||||
|
||||
def submit_phone_code(self, code: str) -> Dict[str, Any]:
|
||||
return self._run(self._submit_phone_code(code))
|
||||
|
||||
async def _submit_password(self, password: str) -> Dict[str, Any]:
|
||||
client = await self._get_client()
|
||||
await client.sign_in(password=password)
|
||||
self._set_state(
|
||||
phase="authorized",
|
||||
status="authorized",
|
||||
details="Telegram session is authorized.",
|
||||
qr_url=None,
|
||||
qr_image=None,
|
||||
)
|
||||
return self.snapshot()
|
||||
|
||||
def submit_password(self, password: str) -> Dict[str, Any]:
|
||||
return self._run(self._submit_password(password))
|
||||
|
||||
|
||||
class ContinuousScrapeManager:
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.Lock()
|
||||
self.thread: Optional[threading.Thread] = None
|
||||
self.stop_event = threading.Event()
|
||||
self.config: Dict[str, Any] = {
|
||||
"enabled": False,
|
||||
"interval_minutes": 1,
|
||||
"channels": [],
|
||||
"run_all_tracked": True,
|
||||
}
|
||||
self.status: Dict[str, Any] = {
|
||||
"running": False,
|
||||
"last_started_at": None,
|
||||
"last_finished_at": None,
|
||||
"last_iteration_at": None,
|
||||
"last_error": None,
|
||||
"logs": [],
|
||||
}
|
||||
|
||||
def _log(self, message: str) -> None:
|
||||
line = f"[{datetime.now().strftime('%H:%M:%S')}] {message}"
|
||||
with self.lock:
|
||||
logs = self.status.setdefault("logs", [])
|
||||
logs.append(line)
|
||||
if len(logs) > 200:
|
||||
del logs[:-200]
|
||||
|
||||
def snapshot(self) -> Dict[str, Any]:
|
||||
with self.lock:
|
||||
return {
|
||||
"config": dict(self.config),
|
||||
"status": {
|
||||
**self.status,
|
||||
"logs": list(self.status.get("logs", [])),
|
||||
},
|
||||
}
|
||||
|
||||
def update(self, enabled: bool, interval_minutes: int, channels: List[str], run_all_tracked: bool) -> Dict[str, Any]:
|
||||
interval_minutes = max(1, int(interval_minutes))
|
||||
with self.lock:
|
||||
self.config = {
|
||||
"enabled": bool(enabled),
|
||||
"interval_minutes": interval_minutes,
|
||||
"channels": channels,
|
||||
"run_all_tracked": bool(run_all_tracked),
|
||||
}
|
||||
if enabled:
|
||||
self.start()
|
||||
else:
|
||||
self.stop()
|
||||
return self.snapshot()
|
||||
|
||||
def start(self) -> None:
|
||||
with self.lock:
|
||||
if self.status["running"]:
|
||||
self._log("Continuous scraping is already running.")
|
||||
return
|
||||
self.stop_event.clear()
|
||||
self.status["running"] = True
|
||||
self.status["last_started_at"] = utc_now_iso()
|
||||
self.status["last_error"] = None
|
||||
self.thread = threading.Thread(target=self._run_loop, daemon=True)
|
||||
self.thread.start()
|
||||
self._log("Continuous scraping started.")
|
||||
|
||||
def stop(self) -> None:
|
||||
self.stop_event.set()
|
||||
with self.lock:
|
||||
was_running = self.status["running"]
|
||||
self.status["running"] = False
|
||||
if was_running:
|
||||
self._log("Continuous scraping stop requested.")
|
||||
|
||||
def _resolve_channels(self) -> List[str]:
|
||||
state = load_state()
|
||||
with self.lock:
|
||||
run_all_tracked = self.config["run_all_tracked"]
|
||||
configured = list(self.config["channels"])
|
||||
if run_all_tracked or not configured:
|
||||
return list(state.get("channels", {}).keys())
|
||||
tracked = set(state.get("channels", {}).keys())
|
||||
return [channel for channel in configured if channel in tracked]
|
||||
|
||||
def _run_loop(self) -> None:
|
||||
while not self.stop_event.is_set():
|
||||
channels = self._resolve_channels()
|
||||
interval_minutes = self.snapshot()["config"]["interval_minutes"]
|
||||
|
||||
if not channels:
|
||||
self._log("No channels configured for continuous scraping.")
|
||||
else:
|
||||
self._log(f"Starting iteration for {len(channels)} channel(s).")
|
||||
buffer = io.StringIO()
|
||||
try:
|
||||
with contextlib.redirect_stdout(buffer), contextlib.redirect_stderr(buffer):
|
||||
run_job("scrape_selected", {"channels": channels})
|
||||
output = buffer.getvalue().strip()
|
||||
if output:
|
||||
for line in output.splitlines():
|
||||
self._log(line)
|
||||
with self.lock:
|
||||
self.status["last_iteration_at"] = utc_now_iso()
|
||||
self.status["last_finished_at"] = utc_now_iso()
|
||||
self.status["last_error"] = None
|
||||
self._log("Iteration finished.")
|
||||
except Exception as exc:
|
||||
output = buffer.getvalue().strip()
|
||||
if output:
|
||||
for line in output.splitlines():
|
||||
self._log(line)
|
||||
self._log(f"Iteration failed: {exc}")
|
||||
with self.lock:
|
||||
self.status["last_error"] = str(exc)
|
||||
|
||||
sleep_seconds = max(5, interval_minutes * 60)
|
||||
self._log(f"Sleeping for {interval_minutes} minute(s).")
|
||||
interrupted = self.stop_event.wait(timeout=sleep_seconds)
|
||||
if interrupted:
|
||||
break
|
||||
|
||||
with self.lock:
|
||||
self.status["running"] = False
|
||||
self._log("Continuous scraping stopped.")
|
||||
|
||||
|
||||
def import_scraper_class():
|
||||
from telegram_scraper_with_forwarding import OptimizedTelegramScraper
|
||||
|
||||
@@ -303,7 +625,9 @@ def run_job(job_type: str, payload: Dict[str, Any]) -> None:
|
||||
scraper = ScraperClass()
|
||||
initialized = await scraper.initialize_client(interactive=False)
|
||||
if not initialized:
|
||||
raise RuntimeError("Telegram client is not ready. Check credentials and session.")
|
||||
raise RuntimeError(
|
||||
"Telegram client is not ready. Check credentials, session, and write access to /app/session."
|
||||
)
|
||||
try:
|
||||
if job_type == "scrape_channel":
|
||||
channel_id = payload["channel_id"]
|
||||
@@ -315,6 +639,11 @@ def run_job(job_type: str, payload: Dict[str, Any]) -> None:
|
||||
for channel_id in channels:
|
||||
offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0)
|
||||
await scraper.scrape_channel(channel_id, offset)
|
||||
elif job_type == "scrape_selected":
|
||||
channels = [str(channel_id) for channel_id in payload.get("channels", [])]
|
||||
for channel_id in channels:
|
||||
offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0)
|
||||
await scraper.scrape_channel(channel_id, offset)
|
||||
elif job_type == "export_all":
|
||||
await scraper.export_data()
|
||||
elif job_type == "rescrape_media":
|
||||
@@ -402,6 +731,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
return self.serve_media(relative)
|
||||
if path == "/api/dashboard":
|
||||
return self.send_json(dashboard_payload(self.app.job_runner))
|
||||
if path == "/api/auth":
|
||||
return self.send_json(self.app.auth_manager.snapshot())
|
||||
if path == "/api/continuous":
|
||||
return self.send_json(self.app.continuous_manager.snapshot())
|
||||
if path == "/api/jobs":
|
||||
return self.send_json(self.app.job_runner.recent_jobs())
|
||||
if path.startswith("/api/jobs/"):
|
||||
@@ -443,7 +776,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
if path.startswith("/media/"):
|
||||
relative = path.removeprefix("/media/")
|
||||
return self.serve_media(relative, head_only=True)
|
||||
if path in {"/api/dashboard", "/api/jobs", "/api/channels"} or path.startswith("/api/jobs/") or (
|
||||
if path in {"/api/dashboard", "/api/auth", "/api/continuous", "/api/jobs", "/api/channels"} or path.startswith("/api/jobs/") or (
|
||||
path.startswith("/api/channels/") and path.endswith("/messages")
|
||||
):
|
||||
self.send_response(HTTPStatus.OK)
|
||||
@@ -468,6 +801,70 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
|
||||
)
|
||||
return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
|
||||
|
||||
if path == "/api/continuous":
|
||||
try:
|
||||
enabled = bool(body.get("enabled"))
|
||||
interval_minutes = int(body.get("interval_minutes", 1))
|
||||
channels = [str(item).strip() for item in body.get("channels", []) if str(item).strip()]
|
||||
run_all_tracked = bool(body.get("run_all_tracked", True))
|
||||
payload = self.app.continuous_manager.update(
|
||||
enabled=enabled,
|
||||
interval_minutes=interval_minutes,
|
||||
channels=channels,
|
||||
run_all_tracked=run_all_tracked,
|
||||
)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_json(payload)
|
||||
|
||||
if path == "/api/auth/credentials":
|
||||
api_id = body.get("api_id")
|
||||
api_hash = str(body.get("api_hash", "")).strip()
|
||||
if not api_id or not api_hash:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "api_id and api_hash are required")
|
||||
try:
|
||||
payload = self.app.auth_manager.save_credentials(int(api_id), api_hash)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_json(payload)
|
||||
|
||||
if path == "/api/auth/qr/start":
|
||||
try:
|
||||
payload = self.app.auth_manager.start_qr_login()
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_json(payload)
|
||||
|
||||
if path == "/api/auth/phone/request":
|
||||
phone = str(body.get("phone", "")).strip()
|
||||
if not phone:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "phone is required")
|
||||
try:
|
||||
payload = self.app.auth_manager.request_phone_code(phone)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_json(payload)
|
||||
|
||||
if path == "/api/auth/phone/submit":
|
||||
code = str(body.get("code", "")).strip()
|
||||
if not code:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "code is required")
|
||||
try:
|
||||
payload = self.app.auth_manager.submit_phone_code(code)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_json(payload)
|
||||
|
||||
if path == "/api/auth/password":
|
||||
password = str(body.get("password", "")).strip()
|
||||
if not password:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, "password is required")
|
||||
try:
|
||||
payload = self.app.auth_manager.submit_password(password)
|
||||
except Exception as exc:
|
||||
return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
return self.send_json(payload)
|
||||
|
||||
if path == "/api/channels/add":
|
||||
channel_id = str(body.get("channel_id", "")).strip()
|
||||
if not channel_id:
|
||||
@@ -605,6 +1002,8 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
|
||||
def __init__(self, server_address: tuple[str, int]):
|
||||
super().__init__(server_address, TelegramScraperRequestHandler)
|
||||
self.job_runner = JobRunner()
|
||||
self.auth_manager = TelegramAuthManager()
|
||||
self.continuous_manager = ContinuousScrapeManager()
|
||||
|
||||
|
||||
def run_server(host: str = DEFAULT_HOST, port: int = DEFAULT_PORT) -> None:
|
||||
|
||||
Reference in New Issue
Block a user