Merge branch 'dev'
ci / lint-prettier (push) Successful in 7s
ci / lint-ruff (push) Successful in 4s
ci / lint-yaml (push) Successful in 5s
ci / validate (push) Successful in 4s
ci / lint-dockerfiles (push) Successful in 4s
ci / publish (push) Successful in 8s

This commit is contained in:
2026-06-28 01:04:17 +02:00
20 changed files with 2932 additions and 514 deletions
+171
View File
@@ -0,0 +1,171 @@
name: ci
'on':
push:
branches:
- '**'
pull_request:
workflow_dispatch:
env:
# Shared image coordinates for the publish job.
REGISTRY: gcr.forust.xyz
IMAGE_NAME: forust/telegram-scraper
jobs:
lint-prettier:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check formatting with Prettier
shell: bash
run: |
mapfile -t prettier_files < <(
git ls-files \
| grep -E '\.(md|json|ya?ml|html|css)$' \
| grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
)
if [ "${#prettier_files[@]}" -eq 0 ]; then
echo "No Prettier-managed files found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
node:22-alpine \
sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
lint-ruff:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Python with Ruff
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/astral-sh/ruff:latest \
check .
lint-yaml:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint YAML syntax
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
cytopia/yamllint:latest \
-c .yamllint .
lint-dockerfiles:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Dockerfiles
shell: bash
run: |
mapfile -t dockerfiles < <(
git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
)
if [ "${#dockerfiles[@]}" -eq 0 ]; then
echo "No Dockerfiles found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
--entrypoint hadolint \
hadolint/hadolint:latest-debian \
-c .hadolint.yaml "${dockerfiles[@]}"
validate:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Validate Kubernetes manifests
shell: bash
run: |
mapfile -t manifests < <(
git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
)
if [ "${#manifests[@]}" -eq 0 ]; then
echo "No Kubernetes manifests found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/yannh/kubeconform:latest \
-strict \
-ignore-missing-schemas \
-summary \
"${manifests[@]}"
publish:
needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
if: github.event_name != 'pull_request'
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Read project version
id: version
shell: bash
run: |
version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')"
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Log in to registry
shell: bash
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-u "${{ secrets.REGISTRY_USERNAME }}" \
--password-stdin
- name: Build and push image
shell: bash
run: |
image="${REGISTRY}/${IMAGE_NAME}"
tags=("latest" "${{ steps.version.outputs.version }}")
case "${GITHUB_REF_NAME}" in
main)
tags+=("main" "prod")
;;
dev)
tags+=("dev")
;;
esac
build_args=()
for tag in "${tags[@]}"; do
build_args+=(-t "${image}:${tag}")
done
docker build "${build_args[@]}" .
for tag in "${tags[@]}"; do
docker push "${image}:${tag}"
done
+171
View File
@@ -0,0 +1,171 @@
name: ci
'on':
push:
branches:
- '**'
pull_request:
workflow_dispatch:
env:
# Shared image coordinates for the publish job.
REGISTRY: gcr.forust.xyz
IMAGE_NAME: forust/telegram-scraper
jobs:
lint-prettier:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check formatting with Prettier
shell: bash
run: |
mapfile -t prettier_files < <(
git ls-files \
| grep -E '\.(md|json|ya?ml|html|css)$' \
| grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
)
if [ "${#prettier_files[@]}" -eq 0 ]; then
echo "No Prettier-managed files found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
node:22-alpine \
sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
lint-ruff:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Python with Ruff
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/astral-sh/ruff:latest \
check .
lint-yaml:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint YAML syntax
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
cytopia/yamllint:latest \
-c .yamllint .
lint-dockerfiles:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Dockerfiles
shell: bash
run: |
mapfile -t dockerfiles < <(
git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
)
if [ "${#dockerfiles[@]}" -eq 0 ]; then
echo "No Dockerfiles found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
--entrypoint hadolint \
hadolint/hadolint:latest-debian \
-c .hadolint.yaml "${dockerfiles[@]}"
validate:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Validate Kubernetes manifests
shell: bash
run: |
mapfile -t manifests < <(
git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
)
if [ "${#manifests[@]}" -eq 0 ]; then
echo "No Kubernetes manifests found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/yannh/kubeconform:latest \
-strict \
-ignore-missing-schemas \
-summary \
"${manifests[@]}"
publish:
needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
if: github.event_name != 'pull_request'
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Read project version
id: version
shell: bash
run: |
version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')"
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Log in to registry
shell: bash
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-u "${{ secrets.REGISTRY_USERNAME }}" \
--password-stdin
- name: Build and push image
shell: bash
run: |
image="${REGISTRY}/${IMAGE_NAME}"
tags=("latest" "${{ steps.version.outputs.version }}")
case "${GITHUB_REF_NAME}" in
main)
tags+=("main" "prod")
;;
dev)
tags+=("dev")
;;
esac
build_args=()
for tag in "${tags[@]}"; do
build_args+=(-t "${image}:${tag}")
done
docker build "${build_args[@]}" .
for tag in "${tags[@]}"; do
docker push "${image}:${tag}"
done
+1 -1
View File
@@ -142,7 +142,7 @@ kubectl apply -f k8s/telegram-scraper.yaml
Then open:
```text
http://<server-ip>:8080
http://<server-ip>:7887
```
### Volumes (Docker)
+1 -1
View File
@@ -25,7 +25,7 @@ ACCOUNT_DEFAULTS: Dict[str, Any] = {
"scrape_media": True,
"forwarding_rules": [],
"continuous_scraping": {
"enabled": True,
"enabled": False,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
+10 -4
View File
@@ -5,13 +5,19 @@ services:
dockerfile: Dockerfile
container_name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
user: 1000:1000
restart: unless-stopped
tty: true
user: '1000:1000'
restart: unless-stopped
ports:
- "7887:8080"
- 7887:8080
healthcheck:
test: ["CMD", "python", "-c", "import http.client,json;c=http.client.HTTPConnection('localhost',8080);c.request('GET','/health');r=c.getresponse();exit(0)if json.loads(r.read()).get('ok')else exit(1)"]
test:
[
CMD,
python,
-c,
"import http.client, json; c=http.client.HTTPConnection('localhost', 8080); c.request('GET', '/health'); r=c.getresponse(); exit(0) if json.loads(r.read()).get('ok') else exit(1)",
]
interval: 30s
timeout: 10s
retries: 3
+15 -1
View File
@@ -13,6 +13,14 @@ def health_payload(
job_queue_size: int,
account_ids: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""
Return a health-check payload for the application.
NOTE: This endpoint does NOT support message search or query parameters.
Health concerns itself with filesystem, database connectivity, job queue,
and account session state. Message search is handled by the message API
endpoints (/api/channels/*/messages, /api/accounts/*/channels/*/messages).
"""
checks = {
"data_dir": _dir_check(data_dir, writable=True),
"session_dir": _dir_check(session_dir, writable=True),
@@ -35,7 +43,13 @@ def health_payload(
"path": str(session_file),
},
}
checks["accounts"] = account_checks
checks["accounts"] = {
"ok": all(
item.get("data_dir", {}).get("ok", False)
for item in account_checks.values()
),
"items": account_checks,
}
ok = all(item.get("ok", False) for item in checks.values())
return {"ok": ok, "status": "ok" if ok else "degraded", "checks": checks}
+5 -3
View File
@@ -34,8 +34,8 @@ spec:
containers:
- name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
tty: true
stdin: true
tty: true
ports:
- containerPort: 8080
livenessProbe:
@@ -59,14 +59,16 @@ spec:
- metadata:
name: telegram-scraper-data
spec:
accessModes: ["ReadWriteOnce"]
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
- metadata:
name: telegram-scraper-session
spec:
accessModes: ["ReadWriteOnce"]
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Mi
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "telegram-scraper"
version = "0.1.0"
version = "2.0.0"
description = "Telegram-scraper with webui"
readme = "README.md"
requires-python = ">=3.11"
-3
View File
@@ -23,11 +23,8 @@ from telethon.errors import FloodWaitError, SessionPasswordNeededError
import qrcode
from app_state import (
StateStore,
account_data_dir,
account_session_path,
get_account_store,
load_account,
save_account,
)
warnings.filterwarnings(
+604
View File
@@ -0,0 +1,604 @@
"""
Smoke / integration tests for the multi-account Telegram scraper core logic.
These tests do NOT require Telegram credentials or network access.
They verify:
- Account creation and state isolation
- Account import safety (no auto-start continuous)
- Account export round-trip
- Account deletion (sidecar file cleanup)
- Active account persistence helpers
- Message search SQL construction
- Health payload structure
- Continuous orchestrator worker deduplication
- load_messages() pagination with search filter
"""
import os
import shutil
import sqlite3
import sys
import tempfile
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
from unittest.mock import MagicMock
# ── Mock heavy dependencies before any webui_server import ─────────────
# We need to mock qrcode and telethon because they aren't installed in CI.
sys.modules["qrcode"] = MagicMock()
sys.modules["qrcode.image"] = MagicMock()
sys.modules["qrcode.image.svg"] = MagicMock()
sys.modules["telethon"] = MagicMock()
sys.modules["telethon.errors"] = MagicMock()
sys.modules["telethon.errors"].SessionPasswordNeededError = type("SessionPasswordNeededError", (Exception,), {})
sys.modules["telegram_scraper_with_forwarding"] = MagicMock()
sys.modules["telegram_scraper_with_forwarding"]._ensure_session_wal = lambda p: None
sys.modules["telegram_scraper_with_forwarding"].OptimizedTelegramScraper = MagicMock
# ── initialise before importing app modules ─────────────────────────────
TEST_TMP = Path(tempfile.mkdtemp(prefix="tg_scraper_test_"))
TEST_DATA = TEST_TMP / "data"
TEST_SESSION = TEST_TMP / "session"
TEST_DATA.mkdir(parents=True, exist_ok=True)
TEST_SESSION.mkdir(parents=True, exist_ok=True)
os.environ["TELEGRAM_SCRAPER_HOST"] = "127.0.0.1"
os.environ["TELEGRAM_SCRAPER_PORT"] = "0"
os.environ["TELEGRAM_SCRAPER_START_CONTINUOUS"] = "0"
# Now safe to import app modules
import app_state # noqa: E402
from app_state import ( # noqa: E402
ACCOUNT_DEFAULTS,
account_data_dir,
account_exists,
account_session_path,
get_account_store,
get_global_store,
list_accounts,
load_account,
)
from health import health_payload # noqa: E402
def _init_test_env():
"""Reset global caches and re-initialise for a clean test."""
app_state._GLOBAL_STORE = None
with app_state._account_stores_lock:
app_state._ACCOUNT_STORES.clear()
_init_test_env()
# ── Helpers ────────────────────────────────────────────────────────────────
def make_account_id() -> str:
return f"test-{int(time.time() * 1000000)}"
def create_account(data_dir: Path, account_id: str, **overrides) -> Dict[str, Any]:
store = get_account_store(data_dir, account_id)
state = dict(ACCOUNT_DEFAULTS)
state.update(overrides)
state["label"] = overrides.get("label", account_id)
store.save(state)
def mutate(g: Dict[str, Any]) -> None:
accounts = g.setdefault("accounts", [])
if account_id not in accounts:
accounts.append(account_id)
get_global_store(data_dir).update(mutate)
return load_account(data_dir, account_id)
def create_channel_db(data_dir: Path, account_id: Optional[str], channel_id: str, messages: List[Dict[str, Any]]) -> None:
if account_id:
db_dir = account_data_dir(data_dir, account_id) / channel_id
else:
db_dir = data_dir / channel_id
db_dir.mkdir(parents=True, exist_ok=True)
db_path = db_dir / f"{channel_id}.db"
conn = sqlite3.connect(str(db_path))
conn.execute(
"""CREATE TABLE IF NOT EXISTS messages (
message_id INTEGER PRIMARY KEY,
date TEXT, sender_id INTEGER,
first_name TEXT, last_name TEXT, username TEXT,
message TEXT, media_type TEXT, media_path TEXT,
reply_to INTEGER, post_author TEXT,
views INTEGER, forwards INTEGER, reactions TEXT
)"""
)
for m in messages:
conn.execute(
"INSERT OR IGNORE INTO messages (message_id, date, message) VALUES (?, ?, ?)",
(m["message_id"], m.get("date"), m.get("message")),
)
conn.commit()
conn.close()
# ── Fixture setup / teardown ────────────────────────────────────────────────
def setup_function():
_init_test_env()
for child in TEST_DATA.iterdir():
if child.is_dir():
shutil.rmtree(child, ignore_errors=True)
else:
child.unlink(missing_ok=True)
def teardown_function():
_init_test_env()
# ═══════════════════════════════════════════════════════════════════════════
# Tests
# ═══════════════════════════════════════════════════════════════════════════
class TestAccountLifecycle:
"""Account creation, isolation, import, export, deletion."""
def test_create_account_minimal(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
assert account_exists(TEST_DATA, aid)
assert aid in list_accounts(TEST_DATA)
state = load_account(TEST_DATA, aid)
assert state["label"] == aid
assert state["continuous_scraping"]["enabled"] is False
def test_account_state_isolation(self):
a1, a2 = make_account_id(), make_account_id()
create_account(TEST_DATA, a1, label="Alpha", api_id=111)
create_account(TEST_DATA, a2, label="Beta", api_id=222)
s1 = load_account(TEST_DATA, a1)
s2 = load_account(TEST_DATA, a2)
assert s1["label"] == "Alpha"
assert s1["api_id"] == 111
assert s2["label"] == "Beta"
assert s2["api_id"] == 222
# channels are segregated — mutate one, verify other unchanged
s1.setdefault("channels", {})["-100aaa"] = 1
get_account_store(TEST_DATA, a1).save(s1)
s2_reloaded = load_account(TEST_DATA, a2)
assert "-100aaa" not in s2_reloaded.get("channels", {})
def test_delete_account_cleans_session_sidecars(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
session_path = Path(account_session_path(TEST_SESSION, aid))
session_path.write_text("session-data")
Path(str(session_path) + "-wal").write_text("wal")
Path(str(session_path) + "-shm").write_text("shm")
Path(str(session_path) + "-journal").write_text("journal")
acc_dir = account_data_dir(TEST_DATA, aid)
if acc_dir.exists():
shutil.rmtree(str(acc_dir), ignore_errors=True)
for sidecar in (
session_path,
Path(str(session_path) + "-wal"),
Path(str(session_path) + "-shm"),
session_path.with_suffix(session_path.suffix + "-journal"),
):
if sidecar.exists():
sidecar.unlink()
assert not session_path.exists()
assert not Path(str(session_path) + "-wal").exists()
assert not Path(str(session_path) + "-shm").exists()
assert not Path(str(session_path) + "-journal").exists()
assert not acc_dir.exists()
def test_delete_account_skips_missing_sidecars(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
session_path = Path(account_session_path(TEST_SESSION, aid))
session_path.write_text("session-only")
acc_dir = account_data_dir(TEST_DATA, aid)
if acc_dir.exists():
shutil.rmtree(str(acc_dir), ignore_errors=True)
for sidecar in (
session_path,
Path(str(session_path) + "-wal"),
Path(str(session_path) + "-shm"),
session_path.with_suffix(session_path.suffix + "-journal"),
):
if sidecar.exists():
sidecar.unlink()
assert not session_path.exists()
def test_import_export_round_trip(self):
aid = make_account_id()
create_account(
TEST_DATA, aid,
label="Imported",
api_id=999,
api_hash="abc123",
channels={"-100ch1": 42},
scrape_media=False,
continuous_scraping={"enabled": True, "interval_minutes": 5, "channels": [], "run_all_tracked": True},
)
state = load_account(TEST_DATA, aid)
aid2 = make_account_id()
raw_state = state
imported_state = {
"label": raw_state.get("label", aid2),
"api_id": raw_state.get("api_id"),
"api_hash": raw_state.get("api_hash"),
"channels": raw_state.get("channels", {}),
"channel_names": raw_state.get("channel_names", {}),
"scrape_media": bool(raw_state.get("scrape_media", True)),
"forwarding_rules": raw_state.get("forwarding_rules", []),
"continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else {
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
},
}
get_account_store(TEST_DATA, aid2).save(imported_state)
def mutate_global(g):
g.setdefault("accounts", []).append(aid2)
get_global_store(TEST_DATA).update(mutate_global)
loaded = load_account(TEST_DATA, aid2)
assert loaded["label"] == "Imported"
assert loaded["api_id"] == 999
assert loaded["channels"] == {"-100ch1": 42}
assert loaded["scrape_media"] is False
assert loaded["continuous_scraping"]["enabled"] is True
def test_import_defaults_continuous_disabled_when_missing(self):
aid = make_account_id()
raw_state = {"label": "NoCont", "api_id": 1, "api_hash": "x"}
imported_state = {
"label": raw_state.get("label", aid),
"api_id": raw_state.get("api_id"),
"api_hash": raw_state.get("api_hash"),
"channels": {},
"channel_names": {},
"scrape_media": True,
"forwarding_rules": [],
"continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else {
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
},
}
get_account_store(TEST_DATA, aid).save(imported_state)
def mutate_global(g):
g.setdefault("accounts", []).append(aid)
get_global_store(TEST_DATA).update(mutate_global)
loaded = load_account(TEST_DATA, aid)
assert loaded["continuous_scraping"]["enabled"] is False
assert loaded["continuous_scraping"]["run_all_tracked"] is True
class TestContinuousOrchestrator:
"""Continuous scraping manager safety using mocked webui_server."""
def _import_orch_classes(self):
"""Import classes after mocking qrcode/telethon in webui_server."""
import webui_server as ws_module
return ws_module.PerAccountContinuousScrapeManager, ws_module.ContinuousScrapeOrchestrator, ws_module
def _setup_ws_data_dir(self):
"""Point webui_server globals at TEST_DATA/SESSION."""
import webui_server as ws_module
self._ws_orig_data = ws_module.DATA_DIR
self._ws_orig_session = ws_module.SESSION_DIR
ws_module.DATA_DIR = TEST_DATA
ws_module.SESSION_DIR = TEST_SESSION
ws_module.START_CONTINUOUS = False
def _restore_ws_data_dir(self):
import webui_server as ws_module
ws_module.DATA_DIR = self._ws_orig_data
ws_module.SESSION_DIR = self._ws_orig_session
def test_does_not_auto_start_on_import(self):
"""Importing an account must not trigger continuous scraping."""
PerAccountContinuousScrapeManager, ContinuousScrapeOrchestrator, ws = self._import_orch_classes()
self._setup_ws_data_dir()
try:
orch = ContinuousScrapeOrchestrator()
aid = make_account_id()
create_account(TEST_DATA, aid, continuous_scraping={
"enabled": True, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
})
orch.add_account(aid, auto_start=False)
snap = orch.snapshot_for(aid)
assert snap["status"]["running"] is False
finally:
self._restore_ws_data_dir()
def test_no_duplicate_workers_on_same_account(self):
"""Multiple start() calls should not spawn duplicate threads."""
PerAccountContinuousScrapeManager, _, ws = self._import_orch_classes()
self._setup_ws_data_dir()
try:
aid = make_account_id()
# Must set enabled=True or refresh_config() will stop the thread
create_account(TEST_DATA, aid, continuous_scraping={
"enabled": True, "interval_minutes": 60, "channels": [], "run_all_tracked": True,
})
mgr = PerAccountContinuousScrapeManager(aid)
# Override refresh_config to prevent auth check from interfering
original_refresh = mgr.refresh_config
mgr.refresh_config = lambda: None
mgr.start()
t1 = mgr.thread
mgr.start() # second call — should be no-op
t2 = mgr.thread
assert t1 is t2, "start() spawned a second thread"
# Also verify only one thread is alive for this manager
alive_count = sum(
1 for t in threading.enumerate()
if t is t1 or t is t2
)
assert alive_count <= 1, "duplicate threads detected"
mgr.stop()
mgr.refresh_config = original_refresh
finally:
self._restore_ws_data_dir()
def test_disabled_account_not_started(self):
"""start_all() must not start accounts with enabled=False."""
_, ContinuousScrapeOrchestrator, ws = self._import_orch_classes()
self._setup_ws_data_dir()
try:
orch = ContinuousScrapeOrchestrator()
aid = make_account_id()
create_account(TEST_DATA, aid, continuous_scraping={
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
})
orch.start_all()
snap = orch.snapshot_for(aid)
assert snap["status"]["running"] is False
finally:
self._restore_ws_data_dir()
class TestMessageSearch:
"""load_messages() SQL correctness with pagination and search."""
def _import_load_messages(self):
import webui_server as ws_module
# Point at TEST_DATA
self._ws_orig = ws_module.DATA_DIR
ws_module.DATA_DIR = TEST_DATA
return ws_module.load_messages
def _restore(self):
import webui_server as ws_module
ws_module.DATA_DIR = self._ws_orig
def test_search_filters_correctly(self):
channel_id = "-100searchtest"
messages = [
{"message_id": 1, "date": "2024-01-01", "message": "hello world"},
{"message_id": 2, "date": "2024-01-02", "message": "foo bar baz"},
{"message_id": 3, "date": "2024-01-03", "message": "hello again"},
]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=100, search="hello")
finally:
self._restore()
texts = [m["text"] for m in result]
assert "hello world" in texts
assert "hello again" in texts
assert "foo bar baz" not in texts
def test_before_id_pagination_works(self):
channel_id = "-100pagination"
messages = [
{"message_id": i, "date": f"2024-01-{i:02d}", "message": f"msg-{i}"}
for i in range(1, 21)
]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=5, before_message_id=15)
finally:
self._restore()
assert len(result) == 5
ids = [m["message_id"] for m in result]
assert all(i < 15 for i in ids)
# load_messages returns rows in ascending message_id order (reversed from DESC query)
assert ids == [10, 11, 12, 13, 14]
def test_search_with_pagination(self):
channel_id = "-100searchpages"
messages = [
{"message_id": i, "date": f"2024-01-{i:02d}", "message": f"hello-{i}" if i % 2 else f"other-{i}"}
for i in range(1, 21)
]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=3, before_message_id=15, search="hello")
finally:
self._restore()
# IDs matching "hello-" below 15: 13, 11, 9, 7, 5, 3, 1 → LIMIT 3 → [9, 11, 13] (ascending)
ids = [m["message_id"] for m in result]
assert ids == [9, 11, 13], f"got {ids}"
assert all("hello" in m["text"] for m in result)
def test_empty_result_no_error(self):
channel_id = "-100emptysearch"
messages = [{"message_id": 1, "date": "2024-01-01", "message": "only one"}]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=100, search="nonexistent")
finally:
self._restore()
assert result == []
class TestHealthPayload:
"""Health endpoint structure."""
def test_health_payload_structure(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
payload = health_payload(
TEST_DATA,
TEST_SESSION,
get_global_store(TEST_DATA),
{},
0,
[aid],
)
assert "ok" in payload
assert "status" in payload
assert "checks" in payload
checks = payload["checks"]
assert "data_dir" in checks
assert "session_dir" in checks
assert "accounts" in checks
assert "items" in checks["accounts"]
assert aid in checks["accounts"]["items"]
def test_health_accounts_wrapper(self):
a1, a2 = make_account_id(), make_account_id()
create_account(TEST_DATA, a1)
create_account(TEST_DATA, a2)
payload = health_payload(
TEST_DATA, TEST_SESSION,
get_global_store(TEST_DATA),
{}, 0, [a1, a2],
)
acc_checks = payload["checks"]["accounts"]
assert acc_checks["ok"] is True
assert len(acc_checks["items"]) == 2
class TestPersistence:
"""UI persistence helpers."""
def test_active_account_localstorage(self):
"""Verifies the save/load logic pattern used in app.js."""
accounts = [
{"id": "work", "label": "Work"},
{"id": "home", "label": "Home"},
]
saved_id = "work"
loaded = None
if saved_id and any(a["id"] == saved_id for a in accounts):
loaded = saved_id
if not loaded:
loaded = next((a["id"] for a in accounts), None)
assert loaded == "work"
saved_id = "longgone"
loaded = None
if saved_id and any(a["id"] == saved_id for a in accounts):
loaded = saved_id
if not loaded and accounts:
loaded = accounts[0]["id"]
assert loaded == "work"
def test_viewer_fallback_on_missing_account(self):
"""Viewer's loadViewerAccount fallback logic."""
accounts = [
{"id": "alpha", "label": "Alpha"},
{"id": "beta", "label": "Beta"},
]
requested = "nonexistent"
requested_exists = requested and any(a["id"] == requested for a in accounts)
account_id = requested if requested_exists else (accounts[0]["id"] if accounts else None)
assert account_id == "alpha"
account_id2 = requested if (requested and any(a["id"] == requested for a in [])) else None
assert account_id2 is None
class TestJobDeduplicationSchema:
"""JobRunner create_job dedup logic."""
def _import_job_runner(self):
import webui_server as ws_module
return ws_module.JobRunner
def test_create_job_dedup_by_account(self):
JobRunner = self._import_job_runner()
runner = JobRunner()
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
assert j1.job_id == j2.job_id, "duplicate job was created instead of being reused"
assert "Reused existing active job" in j2.logs
def test_create_job_allows_different_accounts(self):
JobRunner = self._import_job_runner()
runner = JobRunner()
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
time.sleep(0.002) # ensure different timestamp -> different job_id
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc2"})
assert j1.job_id != j2.job_id
def test_create_job_allows_after_previous_completes(self):
JobRunner = self._import_job_runner()
runner = JobRunner()
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
j1.status = "completed"
time.sleep(0.002) # ensure different timestamp -> different job_id
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
assert j1.job_id != j2.job_id
class TestAccountHealthSummary:
"""account_health_summary structure."""
def _import_health_summary(self):
import webui_server as ws_module
self._ws_orig_data = ws_module.DATA_DIR
self._ws_orig_session = ws_module.SESSION_DIR
ws_module.DATA_DIR = TEST_DATA
ws_module.SESSION_DIR = TEST_SESSION
return ws_module.account_health_summary, ws_module.JobRunner, ws_module
def test_health_summary_structure(self):
account_health_summary, JobRunner, ws = self._import_health_summary()
aid = make_account_id()
create_account(TEST_DATA, aid, api_id=1, api_hash="x")
try:
runner = JobRunner()
health = account_health_summary(aid, runner)
finally:
ws.DATA_DIR = self._ws_orig_data
ws.SESSION_DIR = self._ws_orig_session
assert health["account_id"] == aid
assert "label" in health
assert "data_dir_exists" in health
assert "session_ready" in health
assert "api_credentials" in health
assert health["api_credentials"] is True
assert "channel_count" in health
assert "message_count" in health
assert "media_count" in health
# ── Cleanup all temp data ──────────────────────────────────────────────────
def cleanup_test_data():
if TEST_TMP.exists():
shutil.rmtree(str(TEST_TMP), ignore_errors=True)
import atexit # noqa: E402
atexit.register(cleanup_test_data) # noqa: E402
+478 -218
View File
File diff suppressed because it is too large Load Diff
+18 -4
View File
@@ -1,10 +1,10 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Control Panel</title>
<link rel="stylesheet" href="/static/style.css?v=4" />
<link rel="stylesheet" href="/static/style.css?v=6" />
</head>
<body>
<div class="app-shell">
@@ -17,6 +17,7 @@
<nav class="nav-links">
<a class="nav-link active" href="/">Dashboard</a>
<a class="nav-link" href="/settings">Settings</a>
<a class="nav-link" href="/viewer">Message Viewer</a>
<a class="nav-link" href="/swagger">API Docs</a>
<a class="nav-link" href="/health">Health</a>
@@ -33,7 +34,7 @@
<button class="button primary" id="scrape-all-btn">Scrape all channels</button>
<button class="button" id="export-all-btn">Export all data</button>
<button class="button" id="refresh-dialogs-btn">Refresh dialogs</button>
<button class="button" id="open-settings-btn" type="button">Settings</button>
<a class="button" id="open-settings-btn" href="/settings">Settings</a>
</section>
</aside>
@@ -51,7 +52,9 @@
<div class="panel-header">
<div>
<h2>No Accounts Configured</h2>
<p class="muted">Click "Add Account" in Settings to get started, or use the legacy single-account mode.</p>
<p class="muted">
Click "Add Account" in Settings to get started, or use the legacy single-account mode.
</p>
</div>
</div>
</section>
@@ -81,6 +84,10 @@
<input id="add-account-api-hash" name="api_hash" placeholder="API Hash" />
<button class="button primary" type="submit">Add Account</button>
</form>
<label class="import-account-row">
<span class="muted small">Import account settings JSON</span>
<input id="import-account-file" type="file" accept="application/json,.json" />
</label>
</section>
<!-- ── Account Auth ── (shown per account selected in UI) -->
@@ -165,6 +172,11 @@
<div class="section-title">Forwarding rules</div>
<div class="stat-value forwarding-count">0</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Account health</div>
<div class="stat-value stat-compact account-health-label">-</div>
<div class="muted small account-health-detail"></div>
</div>
</section>
<section class="panel">
@@ -247,6 +259,7 @@
<div class="continuous-meta">
<div>Status: <span class="continuous-status">-</span></div>
<div>Last run: <span class="continuous-last-iteration">-</span></div>
<div>Next run: <span class="continuous-next-run">-</span></div>
<div>Last error: <span class="continuous-last-error">-</span></div>
</div>
@@ -300,6 +313,7 @@
</div>
<div class="account-list-actions">
<button class="button button-small account-select-btn">Select</button>
<button class="button button-small account-export-btn">Export</button>
<button class="button button-small danger account-remove-btn">Remove</button>
</div>
</div>
+156
View File
@@ -0,0 +1,156 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Settings</title>
<link rel="stylesheet" href="/static/style.css?v=6" />
</head>
<body>
<div class="app-shell settings-page-shell">
<aside class="sidebar">
<div class="brand-block">
<div class="eyebrow">Telegram Scraper</div>
<h1>Settings</h1>
<p class="muted">Account credentials, import/export, and runtime safety controls.</p>
</div>
<nav class="nav-links">
<a class="nav-link" href="/">Dashboard</a>
<a class="nav-link active" href="/settings">Settings</a>
<a class="nav-link" href="/viewer">Message Viewer</a>
<a class="nav-link" href="/swagger">API Docs</a>
<a class="nav-link" href="/health">Health</a>
</nav>
<section class="status-card">
<div class="section-title">Accounts</div>
<div id="settings-account-list" class="settings-account-list"></div>
</section>
</aside>
<main class="content settings-page">
<section class="panel settings-hero">
<div>
<div class="eyebrow">Account Settings</div>
<h2 id="settings-title">Loading accounts</h2>
<p id="settings-subtitle" class="muted">Select an account to manage credentials and scraper options.</p>
</div>
<div class="action-row">
<button class="button" id="settings-export-btn" type="button" disabled>Export</button>
<button class="button danger" id="settings-delete-btn" type="button" disabled>Delete</button>
</div>
</section>
<section id="settings-empty" class="panel hidden">
<div class="empty-state">
<h2>Add first account</h2>
<p class="muted">Create an account below or import an exported account JSON.</p>
</div>
</section>
<section class="panel-grid settings-health-grid">
<div class="panel stat-panel">
<div class="section-title">Credentials</div>
<div class="stat-value stat-compact" id="health-credentials">-</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Session</div>
<div class="stat-value stat-compact" id="health-session">-</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Continuous</div>
<div class="stat-value stat-compact" id="health-continuous">-</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Last scrape</div>
<div class="stat-value stat-compact" id="health-last-scrape">-</div>
</div>
</section>
<section class="settings-layout">
<div class="panel">
<div class="panel-header">
<div>
<h2>Create or Import</h2>
<p class="muted">New accounts start without continuous scraping enabled.</p>
</div>
</div>
<form id="settings-add-account-form" class="stack-form">
<input id="settings-add-account-id" name="account_id" placeholder="Account ID (e.g. work)" required />
<input id="settings-add-account-label" name="label" placeholder="Display name" />
<input id="settings-add-account-api-id" name="api_id" placeholder="API ID" />
<input id="settings-add-account-api-hash" name="api_hash" placeholder="API Hash" />
<button class="button primary" type="submit">Add Account</button>
</form>
<label class="import-account-row">
<span class="muted small">Import account settings JSON</span>
<input id="settings-import-account-file" type="file" accept="application/json,.json" />
</label>
</div>
<div class="panel">
<div class="panel-header">
<div>
<h2>Credentials & Auth</h2>
<p class="muted" id="settings-auth-label">No account selected.</p>
</div>
</div>
<form id="settings-credentials-form" class="stack-form">
<input id="settings-api-id-input" name="api_id" placeholder="API ID" />
<input id="settings-api-hash-input" name="api_hash" placeholder="API Hash" />
<button class="button" type="submit">Save credentials</button>
</form>
<div class="auth-actions">
<button class="button primary" id="settings-start-qr-btn" type="button">Start QR login</button>
</div>
<div id="settings-qr-wrap" class="qr-wrap hidden">
<img id="settings-qr-image" alt="Telegram QR login" />
<p class="muted small">Open Telegram -> Settings -> Devices -> Scan QR.</p>
</div>
<form id="settings-phone-form" class="stack-form">
<input id="settings-phone-input" name="phone" placeholder="+1234567890" />
<button class="button" type="submit">Send code</button>
</form>
<form id="settings-code-form" class="stack-form hidden">
<input id="settings-code-input" name="code" placeholder="Telegram code" />
<button class="button" type="submit">Confirm code</button>
</form>
<form id="settings-password-form" class="stack-form hidden">
<input id="settings-password-input" name="password" type="password" placeholder="2FA password" />
<button class="button" type="submit">Confirm password</button>
</form>
</div>
<div class="panel">
<div class="panel-header">
<div>
<h2>Scraping</h2>
<p class="muted">Account-level parser settings.</p>
</div>
</div>
<label class="toggle-row">
<span>Download media</span>
<span class="switch">
<input id="settings-scrape-media-toggle" type="checkbox" />
<span class="switch-slider"></span>
</span>
</label>
<div class="settings-meta-list">
<div>Tracked channels: <span id="settings-channel-count">0</span></div>
<div>Messages: <span id="settings-message-count">0</span></div>
<div>Media: <span id="settings-media-count">0</span></div>
</div>
</div>
</section>
</main>
</div>
<script src="/static/settings.js?v=1"></script>
</body>
</html>
+431
View File
@@ -0,0 +1,431 @@
const settingsState = {
accounts: [],
activeAccount: null,
dashboard: null,
auth: null,
continuous: null,
};
const ACTIVE_ACCOUNT_STORAGE_KEY = 'telegramScraper.activeAccount';
async function api(path, options = {}) {
const response = await fetch(path, {
headers: { 'Content-Type': 'application/json' },
...options,
});
const data = await response.json();
if (!response.ok) {
throw new Error(data.error || 'Request failed');
}
return data;
}
function showToast(message, type = 'info') {
let root = document.getElementById('toast-root');
if (!root) {
root = document.createElement('div');
root.id = 'toast-root';
root.className = 'toast-root';
document.body.appendChild(root);
}
const toast = document.createElement('div');
toast.className = `toast toast-${type}`;
toast.textContent = message;
root.appendChild(toast);
window.setTimeout(() => {
toast.classList.add('toast-hide');
window.setTimeout(() => toast.remove(), 250);
}, 3600);
}
function downloadJson(filename, payload) {
const blob = new Blob([JSON.stringify(payload, null, 2) + '\n'], {
type: 'application/json',
});
const url = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
link.download = filename;
document.body.appendChild(link);
link.click();
link.remove();
URL.revokeObjectURL(url);
}
function displayTime(value) {
if (!value) return '-';
const date = new Date(value);
if (Number.isNaN(date.getTime())) return String(value).replace('T', ' ').replace('+00:00', ' UTC');
return date.toLocaleString(undefined, {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
});
}
function isAccountAuthorized(auth) {
const status = auth?.auth_status || auth || {};
return (
auth?.phase === 'authorized' ||
auth?.status === 'authorized' ||
status.status === 'ready' ||
status.status === 'authorized'
);
}
function accountLabel(account) {
return account?.label || account?.id || 'No account';
}
function setActiveAccount(accountId) {
settingsState.activeAccount = accountId || null;
if (accountId) {
localStorage.setItem(ACTIVE_ACCOUNT_STORAGE_KEY, accountId);
} else {
localStorage.removeItem(ACTIVE_ACCOUNT_STORAGE_KEY);
}
}
function chooseInitialAccount() {
const requested = new URLSearchParams(window.location.search).get('account');
if (requested && settingsState.accounts.some((account) => account.id === requested)) {
return requested;
}
const saved = localStorage.getItem(ACTIVE_ACCOUNT_STORAGE_KEY);
if (saved && settingsState.accounts.some((account) => account.id === saved)) {
return saved;
}
return settingsState.accounts[0]?.id || null;
}
function renderAccountList() {
const root = document.getElementById('settings-account-list');
root.innerHTML = '';
if (!settingsState.accounts.length) {
root.innerHTML = '<div class="empty-state">No accounts yet.</div>';
return;
}
settingsState.accounts.forEach((account) => {
const button = document.createElement('button');
button.className = 'settings-account-button';
button.type = 'button';
button.classList.toggle('active', account.id === settingsState.activeAccount);
const name = document.createElement('span');
name.className = 'settings-account-name';
name.textContent = accountLabel(account);
const status = document.createElement('span');
status.className = isAccountAuthorized(account.auth) ? 'settings-account-status ok' : 'settings-account-status';
status.textContent = isAccountAuthorized(account.auth) ? 'Authorized' : 'Needs auth';
button.append(name, status);
button.addEventListener('click', () => loadAccount(account.id));
root.appendChild(button);
});
}
function renderEmptyState() {
const hasAccount = Boolean(settingsState.activeAccount);
document.getElementById('settings-empty').classList.toggle('hidden', hasAccount);
document.getElementById('settings-export-btn').disabled = !hasAccount;
document.getElementById('settings-delete-btn').disabled = !hasAccount;
document
.getElementById('settings-credentials-form')
.querySelectorAll('input, button')
.forEach((node) => {
node.disabled = !hasAccount;
});
document.getElementById('settings-start-qr-btn').disabled = !hasAccount;
document
.getElementById('settings-phone-form')
.querySelectorAll('input, button')
.forEach((node) => {
node.disabled = !hasAccount;
});
document.getElementById('settings-scrape-media-toggle').disabled = !hasAccount;
}
function renderAccountData() {
const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount);
const dashboard = settingsState.dashboard || {};
const health = dashboard.health || {};
const auth = settingsState.auth || account?.auth || {};
const continuous = settingsState.continuous || {};
const continuousStatus = continuous.status || {};
const continuousConfig = continuous.config || {};
document.getElementById('settings-title').textContent = account ? accountLabel(account) : 'No account selected';
document.getElementById('settings-subtitle').textContent = account
? `Account ID: ${account.id}`
: 'Create or import an account to manage settings.';
document.getElementById('settings-auth-label').textContent = account
? `${accountLabel(account)} auth status`
: 'No account selected.';
document.getElementById('health-credentials').textContent = health.api_credentials ? 'Saved' : 'Missing';
document.getElementById('health-credentials').style.color = health.api_credentials ? 'var(--ok)' : 'var(--warn)';
document.getElementById('health-session').textContent =
health.session_ready || isAccountAuthorized(auth) ? 'Ready' : 'Missing';
document.getElementById('health-session').style.color =
health.session_ready || isAccountAuthorized(auth) ? 'var(--ok)' : 'var(--warn)';
document.getElementById('health-continuous').textContent = continuousStatus.running
? 'Running'
: continuousConfig.enabled
? 'Enabled'
: 'Stopped';
document.getElementById('health-continuous').style.color = continuousStatus.running ? 'var(--ok)' : 'var(--dim)';
document.getElementById('health-last-scrape').textContent = displayTime(
health.last_scrape || continuousStatus.last_iteration_at,
);
document.getElementById('settings-scrape-media-toggle').checked = Boolean(dashboard.dashboard?.scrape_media);
document.getElementById('settings-channel-count').textContent = String(health.channel_count || 0);
document.getElementById('settings-message-count').textContent = String(health.message_count || 0);
document.getElementById('settings-media-count').textContent = String(health.media_count || 0);
renderEmptyState();
renderAccountList();
}
async function loadAccounts() {
const payload = await api('/api/accounts');
settingsState.accounts = payload.accounts || [];
setActiveAccount(chooseInitialAccount());
renderAccountList();
if (settingsState.activeAccount) {
await loadAccount(settingsState.activeAccount, { preserveUrl: true });
} else {
settingsState.dashboard = null;
settingsState.auth = null;
settingsState.continuous = null;
renderAccountData();
}
}
async function loadAccount(accountId, options = {}) {
setActiveAccount(accountId);
const [dashboard, auth, continuous] = await Promise.all([
api(`/api/accounts/${encodeURIComponent(accountId)}`),
api(`/api/accounts/${encodeURIComponent(accountId)}/auth`).catch(() => ({})),
api(`/api/accounts/${encodeURIComponent(accountId)}/continuous`).catch(() => ({})),
]);
settingsState.dashboard = dashboard;
settingsState.auth = auth;
settingsState.continuous = continuous;
if (!options.preserveUrl) {
history.replaceState(null, '', `/settings?account=${encodeURIComponent(accountId)}`);
}
renderAccountData();
}
async function createAccount(event) {
event.preventDefault();
const accountId = document.getElementById('settings-add-account-id').value.trim();
const label = document.getElementById('settings-add-account-label').value.trim();
const apiId = document.getElementById('settings-add-account-api-id').value.trim();
const apiHash = document.getElementById('settings-add-account-api-hash').value.trim();
if (!accountId) return;
await api('/api/accounts', {
method: 'POST',
body: JSON.stringify({
account_id: accountId,
label: label || accountId,
api_id: apiId ? parseInt(apiId) : undefined,
api_hash: apiHash || undefined,
}),
});
event.currentTarget.reset();
await loadAccounts();
await loadAccount(accountId);
showToast(`Added ${accountId}.`, 'success');
}
async function importAccount(event) {
const file = event.currentTarget.files?.[0];
if (!file) return;
try {
const payload = JSON.parse(await file.text());
const accountId = payload.account_id || payload.id;
if (!accountId) throw new Error('account_id is missing in import file');
await api('/api/accounts/import', {
method: 'POST',
body: JSON.stringify(payload),
});
event.currentTarget.value = '';
await loadAccounts();
await loadAccount(accountId);
showToast(`Imported ${accountId}.`, 'success');
} catch (err) {
showToast(`Failed to import account: ${err.message}`, 'error');
}
}
async function saveCredentials(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const apiId = document.getElementById('settings-api-id-input').value.trim();
const apiHash = document.getElementById('settings-api-hash-input').value.trim();
if (!apiId || !apiHash) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/credentials`, {
method: 'POST',
body: JSON.stringify({ api_id: apiId, api_hash: apiHash }),
});
await loadAccount(settingsState.activeAccount);
showToast('Credentials saved.', 'success');
}
async function startQrLogin() {
if (!settingsState.activeAccount) return;
const data = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/qr/start`, {
method: 'POST',
body: JSON.stringify({}),
});
if (data.qr_image) {
document.getElementById('settings-qr-image').src = data.qr_image;
document.getElementById('settings-qr-wrap').classList.remove('hidden');
}
await loadAccount(settingsState.activeAccount);
}
async function requestPhoneCode(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const phone = document.getElementById('settings-phone-input').value.trim();
if (!phone) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/request`, {
method: 'POST',
body: JSON.stringify({ phone }),
});
document.getElementById('settings-code-form').classList.remove('hidden');
await loadAccount(settingsState.activeAccount);
}
async function submitPhoneCode(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const code = document.getElementById('settings-code-input').value.trim();
if (!code) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/submit`, {
method: 'POST',
body: JSON.stringify({ code }),
});
document.getElementById('settings-code-input').value = '';
await loadAccount(settingsState.activeAccount);
}
async function submitPassword(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const password = document.getElementById('settings-password-input').value.trim();
if (!password) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/password`, {
method: 'POST',
body: JSON.stringify({ password }),
});
document.getElementById('settings-password-input').value = '';
await loadAccount(settingsState.activeAccount);
}
async function exportAccount() {
if (!settingsState.activeAccount) return;
const payload = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/export`);
downloadJson(`telegram-scraper-account-${settingsState.activeAccount}.json`, payload);
showToast('Account exported.', 'success');
}
async function deleteAccount() {
if (!settingsState.activeAccount) return;
const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount);
if (!window.confirm(`Remove account "${accountLabel(account)}"? All its account data will be deleted.`)) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}`, { method: 'DELETE' });
setActiveAccount(null);
await loadAccounts();
showToast('Account deleted.', 'success');
}
async function toggleMedia(event) {
if (!settingsState.activeAccount) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/settings/media`, {
method: 'POST',
body: JSON.stringify({ value: event.currentTarget.checked }),
});
await loadAccount(settingsState.activeAccount);
}
function bindEvents() {
document.getElementById('settings-add-account-form').addEventListener('submit', async (event) => {
try {
await createAccount(event);
} catch (err) {
showToast(`Failed to add account: ${err.message}`, 'error');
}
});
document.getElementById('settings-import-account-file').addEventListener('change', importAccount);
document.getElementById('settings-credentials-form').addEventListener('submit', async (event) => {
try {
await saveCredentials(event);
} catch (err) {
showToast(`Failed to save credentials: ${err.message}`, 'error');
}
});
document.getElementById('settings-start-qr-btn').addEventListener('click', async () => {
try {
await startQrLogin();
} catch (err) {
showToast(`QR login failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-phone-form').addEventListener('submit', async (event) => {
try {
await requestPhoneCode(event);
} catch (err) {
showToast(`Phone login failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-code-form').addEventListener('submit', async (event) => {
try {
await submitPhoneCode(event);
} catch (err) {
showToast(`Code submit failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-password-form').addEventListener('submit', async (event) => {
try {
await submitPassword(event);
} catch (err) {
showToast(`Password submit failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-export-btn').addEventListener('click', async () => {
try {
await exportAccount();
} catch (err) {
showToast(`Export failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-delete-btn').addEventListener('click', async () => {
try {
await deleteAccount();
} catch (err) {
showToast(`Delete failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-scrape-media-toggle').addEventListener('change', async (event) => {
try {
await toggleMedia(event);
} catch (err) {
showToast(`Media setting failed: ${err.message}`, 'error');
}
});
}
bindEvents();
loadAccounts().catch((err) => {
console.error(err);
showToast(`Failed to load settings: ${err.message}`, 'error');
});
+303 -92
View File
@@ -4,21 +4,24 @@
--panel: #0b0b0b;
--panel-solid: #101010;
--panel-alt: #050505;
--text-color: #f2f3f5;
--accent: #1fb9aa;
--accent-2: #58a6ff;
--bubble-own: #12342f;
--bubble-other: #171717;
--dim: #9aa0a6;
--line: #242424;
--text-color: #f3f5f8;
--accent: #2f57b8;
--accent-hover: #3b63c6;
--accent-soft: #0d1730;
--accent-line: #24428d;
--accent-2: #6f95eb;
--bubble-own: #0f1c3f;
--bubble-other: #111111;
--dim: #9ea7b7;
--line: #1f1f1f;
--danger: #ff7373;
--ok: #65e6a4;
--warn: #f6c969;
--info: #8ec7ff;
--radius: 18px;
--radius: 22px;
--shadow: none;
--font-ui: "Segoe UI", "Helvetica Neue", Arial, sans-serif;
--font-mono: "IBM Plex Mono", "SFMono-Regular", Consolas, monospace;
--font-ui: 'Segoe UI', 'Helvetica Neue', Arial, sans-serif;
--font-mono: 'IBM Plex Mono', 'SFMono-Regular', Consolas, monospace;
}
* {
@@ -27,7 +30,7 @@
body {
margin: 0;
background: #000;
background: var(--bg-color);
color: var(--text-color);
font-family: var(--font-ui);
font-size: 15px;
@@ -42,8 +45,8 @@ a {
a:hover,
button:hover {
border-color: #333;
background: #151515;
border-color: #2b2b2b;
background: #111111;
color: var(--text-color);
}
@@ -58,36 +61,48 @@ button {
height: 100vh;
overflow: hidden;
display: grid;
grid-template-columns: 308px minmax(0, 1fr);
grid-template-columns: 360px minmax(0, 1fr);
}
.sidebar {
background: #050505;
background: var(--panel-alt);
border-right: 1px solid var(--line);
padding: 24px 20px;
padding: 30px 24px;
overflow-y: auto;
}
.content {
overflow-y: auto;
padding: 24px;
padding: 32px;
max-width: 1440px;
width: 100%;
}
.brand-block {
padding: 0 2px;
}
.brand-block h1,
.dialog-header h2 {
margin: 0;
color: var(--accent);
font-size: 1.5rem;
font-size: 1.65rem;
letter-spacing: -0.04em;
}
.brand-block .muted {
margin: 22px 0 0;
max-width: 26ch;
font-size: 1.03rem;
line-height: 1.62;
}
.eyebrow,
.section-title {
color: var(--dim);
font-size: 0.75rem;
text-transform: uppercase;
letter-spacing: 0.08em;
letter-spacing: 0.14em;
}
.muted {
@@ -106,7 +121,7 @@ button {
}
.nav-links {
margin: 28px 0;
margin: 34px 0;
}
.nav-link,
@@ -120,6 +135,12 @@ input,
border-radius: 999px;
}
a.nav-link,
a.button,
a.content-tab {
border-bottom: 1px solid var(--line);
}
.nav-link,
.button,
.content-tab {
@@ -127,24 +148,33 @@ input,
align-items: center;
justify-content: center;
min-height: 42px;
padding: 9px 13px;
padding: 10px 18px;
cursor: pointer;
transition: background-color 0.18s ease, border-color 0.18s ease, transform 0.18s ease;
transition:
background-color 0.18s ease,
border-color 0.18s ease,
box-shadow 0.18s ease,
transform 0.18s ease;
}
.nav-link {
min-height: 52px;
font-size: 1rem;
}
.nav-link.active,
.button.primary,
.content-tab.active {
border-color: #0f6f67;
border-color: var(--accent-line);
color: #ffffff;
background: #0f6f67;
background: var(--accent);
box-shadow: none;
}
.nav-link.active:hover,
.button.primary:hover,
.content-tab.active:hover {
background: #128277;
background: var(--accent-hover);
color: #ffffff;
}
@@ -179,24 +209,35 @@ input,
.panel,
.job-card,
.message-card {
padding: 16px;
padding: 22px;
}
.status-card {
margin-top: 16px;
margin-top: 20px;
display: grid;
gap: 12px;
align-content: start;
}
.status-card .button {
width: 100%;
margin-top: 8px;
margin-top: 0;
}
.status-card .section-title,
.status-card p,
.status-card .status-badge {
margin-top: 0;
margin-bottom: 0;
}
.status-badge {
display: inline-flex;
padding: 6px 10px;
padding: 7px 12px;
border: 1px solid var(--line);
border-radius: 999px;
color: var(--ok);
background: #0d1410;
}
.content-tabs {
@@ -216,13 +257,17 @@ input,
.panel-grid {
display: grid;
gap: 16px;
gap: 18px;
grid-template-columns: repeat(3, minmax(0, 1fr));
margin-bottom: 16px;
margin-bottom: 18px;
}
.settings-health-grid {
grid-template-columns: repeat(4, minmax(0, 1fr));
}
.panel {
margin-bottom: 16px;
margin-bottom: 18px;
}
.panel-header,
@@ -235,7 +280,7 @@ input,
}
.stat-value {
margin-top: 8px;
margin-top: 10px;
font-size: 2.25rem;
font-weight: 700;
letter-spacing: -0.05em;
@@ -257,15 +302,88 @@ input,
margin-top: 12px;
}
.settings-layout {
display: grid;
grid-template-columns: minmax(280px, 0.9fr) minmax(320px, 1.1fr);
gap: 18px;
align-items: start;
}
.settings-layout .panel:last-child {
grid-column: 1 / -1;
}
.settings-page-shell .content {
max-width: 1280px;
}
.settings-hero {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
}
.settings-account-list {
display: grid;
gap: 10px;
margin-top: 14px;
}
.settings-account-button {
width: 100%;
min-height: 48px;
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
gap: 8px;
align-items: center;
text-align: left;
border: 1px solid var(--line);
border-radius: 16px;
background: #101010;
color: var(--text-color);
padding: 12px 14px;
cursor: pointer;
}
.settings-account-button.active {
border-color: var(--accent-line);
background: var(--accent-soft);
}
.settings-account-name {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.settings-account-status {
color: var(--warn);
font-size: 0.78rem;
}
.settings-account-status.ok {
color: var(--ok);
}
.settings-meta-list {
display: grid;
gap: 8px;
margin-top: 16px;
color: var(--dim);
}
input {
min-width: 160px;
padding: 10px 13px;
padding: 11px 14px;
outline: 0;
border-radius: 14px;
border-radius: 16px;
}
input:focus {
border-color: var(--accent);
box-shadow: none;
}
.toggle-row {
@@ -301,7 +419,7 @@ input:focus {
}
.switch-slider::before {
content: "";
content: '';
position: absolute;
top: 4px;
left: 4px;
@@ -309,11 +427,14 @@ input:focus {
height: 16px;
background: var(--dim);
border-radius: 50%;
transition: transform 0.16s ease, background-color 0.16s ease;
transition:
transform 0.16s ease,
background-color 0.16s ease;
}
.switch input:checked + .switch-slider {
border-color: var(--accent);
background: var(--accent-soft);
}
.switch input:checked + .switch-slider::before {
@@ -339,7 +460,7 @@ td {
}
tbody tr:hover {
background: #111;
background: #0d0d0d;
}
.action-row {
@@ -386,10 +507,10 @@ tbody tr:hover {
align-items: center;
gap: 10px;
min-height: 38px;
padding: 8px;
padding: 10px 12px;
border: 1px solid var(--line);
border-radius: 14px;
background: #0f0f0f;
border-radius: 16px;
background: #101010;
}
.checkbox-row input {
@@ -419,7 +540,7 @@ tbody tr:hover {
}
.log-line:hover {
background: #111;
background: rgba(53, 95, 199, 0.08);
}
.log-time,
@@ -460,11 +581,11 @@ tbody tr:hover {
}
.settings-dialog::backdrop {
background: rgba(0, 0, 0, 0.72);
background: rgba(0, 0, 0, 0.82);
}
.dialog-shell {
padding: 18px;
padding: 24px;
}
.settings-section {
@@ -503,12 +624,12 @@ tbody tr:hover {
}
.viewer-sidebar {
background: #050505;
background: var(--panel-alt);
border-right: 1px solid var(--line);
padding: 14px 10px;
padding: 18px 14px;
display: flex;
flex-direction: column;
gap: 12px;
gap: 14px;
overflow-y: auto;
}
@@ -527,13 +648,13 @@ tbody tr:hover {
.viewer-sidebar-head h1 {
margin: 0;
color: var(--accent);
font-size: 1.45rem;
font-size: 1.55rem;
letter-spacing: -0.04em;
}
.viewer-channel-list {
display: grid;
gap: 2px;
gap: 6px;
}
.viewer-channel-item {
@@ -543,23 +664,23 @@ tbody tr:hover {
gap: 10px;
width: 100%;
min-height: 68px;
padding: 8px 10px;
padding: 10px 12px;
text-align: left;
cursor: pointer;
border: 1px solid transparent;
background: transparent;
color: var(--text-color);
border-radius: 12px;
border-radius: 16px;
}
.viewer-channel-item.active {
border-color: transparent;
background: #12342f;
border-color: var(--accent-line);
background: var(--accent-soft);
box-shadow: none;
}
.viewer-channel-item:hover {
background: #111;
background: #0f0f0f;
}
.viewer-channel-avatar {
@@ -569,7 +690,7 @@ tbody tr:hover {
width: 48px;
height: 48px;
border-radius: 50%;
background: #5f6b70;
background: #274894;
color: white;
font-weight: 700;
letter-spacing: -0.03em;
@@ -613,8 +734,8 @@ tbody tr:hover {
min-width: 22px;
padding: 2px 7px;
border-radius: 999px;
background: #12342f;
color: var(--accent);
background: #111a32;
color: var(--accent-2);
font-size: 0.76rem;
text-align: center;
}
@@ -626,31 +747,56 @@ tbody tr:hover {
flex-direction: column;
height: 100vh;
overflow: hidden;
background:
radial-gradient(circle at 20px 20px, rgba(255, 255, 255, 0.025) 1px, transparent 1.5px),
radial-gradient(circle at 58px 54px, rgba(255, 255, 255, 0.018) 1px, transparent 1.5px),
#000;
background-size: auto, 84px 84px, 84px 84px, auto;
background: var(--bg-color);
}
.viewer-header {
flex-shrink: 0;
padding: 16px 24px;
padding: 18px 26px;
border-bottom: 1px solid var(--line);
background: #050505;
background: var(--panel-alt);
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
}
.viewer-header h2 {
margin: 0;
color: var(--accent);
font-size: 1.45rem;
font-size: 1.55rem;
letter-spacing: -0.04em;
}
.viewer-tools {
display: flex;
align-items: center;
justify-content: flex-end;
gap: 8px;
flex-wrap: wrap;
}
.viewer-tools input[type='search'] {
min-width: min(260px, 40vw);
}
.viewer-auto-refresh {
display: inline-flex;
align-items: center;
gap: 6px;
min-height: 34px;
color: var(--dim);
font-size: 0.82rem;
}
.viewer-auto-refresh input {
min-width: auto;
}
.messages-list {
flex: 1;
overflow-y: auto;
padding: 18px 34px 22px;
padding: 22px 38px 26px;
display: flex;
flex-direction: column;
}
@@ -676,7 +822,7 @@ tbody tr:hover {
.chat-date-sep::before,
.chat-date-sep::after {
content: "";
content: '';
flex: 1;
height: 1px;
background: #242424;
@@ -686,7 +832,7 @@ tbody tr:hover {
align-self: center;
padding: 5px 12px;
border-radius: 999px;
background: #111;
background: #111111;
}
.chat-date-sep:empty {
@@ -709,11 +855,11 @@ tbody tr:hover {
.chat-bubble {
max-width: min(680px, 74%);
padding: 8px 12px 7px;
padding: 9px 13px 8px;
background: var(--bubble-other);
border: 1px solid #242424;
border: 1px solid var(--line);
border-radius: 18px 18px 18px 6px;
margin-bottom: 6px;
margin-bottom: 7px;
overflow-wrap: anywhere;
word-break: break-word;
position: relative;
@@ -721,7 +867,7 @@ tbody tr:hover {
}
.chat-bubble::before {
content: "";
content: '';
position: absolute;
top: 8px;
border: 6px solid transparent;
@@ -741,7 +887,7 @@ tbody tr:hover {
.chat-bubble-wrap--own .chat-bubble {
background: var(--bubble-own);
border-color: #1f5b52;
border-color: var(--accent-line);
border-radius: 18px 18px 6px 18px;
}
@@ -750,8 +896,13 @@ tbody tr:hover {
}
@keyframes chat-highlight {
0%, 100% { box-shadow: 0 0 0 0 rgba(139, 180, 248, 0); }
50% { box-shadow: 0 0 0 3px rgba(139, 180, 248, 0.4); }
0%,
100% {
box-shadow: 0 0 0 0 rgba(139, 180, 248, 0);
}
50% {
box-shadow: 0 0 0 3px rgba(139, 180, 248, 0.4);
}
}
/* ---------- Sender ---------- */
@@ -852,6 +1003,55 @@ tbody tr:hover {
display: none;
}
.empty-state,
.viewer-empty-state {
padding: 16px;
color: var(--dim);
border: 1px solid var(--line);
border-radius: 14px;
background: #0b0b0b;
}
.viewer-empty-state {
align-self: center;
margin-top: 16px;
}
.toast-root {
position: fixed;
right: 18px;
bottom: 18px;
z-index: 1000;
display: grid;
gap: 8px;
width: min(360px, calc(100vw - 36px));
}
.toast {
padding: 12px 14px;
border: 1px solid var(--line);
border-radius: 14px;
background: var(--panel-solid);
color: var(--text-color);
transition:
opacity 0.2s ease,
transform 0.2s ease;
}
.toast-success {
border-color: var(--accent-line);
}
.toast-error {
border-color: #7a2a2a;
color: #ffd4d4;
}
.toast-hide {
opacity: 0;
transform: translateY(8px);
}
.api-docs {
display: grid;
gap: 16px;
@@ -927,9 +1127,9 @@ tbody tr:hover {
.account-tabs {
display: flex;
flex-wrap: wrap;
gap: 8px;
margin-bottom: 16px;
padding-bottom: 10px;
gap: 10px;
margin-bottom: 18px;
padding-bottom: 14px;
border-bottom: 1px solid var(--line);
}
@@ -937,8 +1137,8 @@ tbody tr:hover {
display: inline-flex;
align-items: center;
gap: 8px;
min-height: 38px;
padding: 7px 14px;
min-height: 40px;
padding: 8px 15px;
border: 1px solid var(--line);
border-radius: 999px;
background: #101010;
@@ -949,9 +1149,10 @@ tbody tr:hover {
}
.account-tab.active {
border-color: #0f6f67;
border-color: var(--accent-line);
color: #ffffff;
background: #0f6f67;
background: var(--accent);
box-shadow: none;
}
.account-tab .account-tab-status {
@@ -981,8 +1182,8 @@ tbody tr:hover {
.accounts-list {
display: grid;
gap: 8px;
margin-bottom: 12px;
gap: 10px;
margin-bottom: 14px;
}
.account-list-item {
@@ -990,10 +1191,10 @@ tbody tr:hover {
align-items: center;
justify-content: space-between;
gap: 12px;
padding: 10px 12px;
padding: 12px 14px;
border: 1px solid var(--line);
border-radius: 16px;
background: #0f0f0f;
background: #101010;
}
.account-list-info {
@@ -1022,12 +1223,22 @@ tbody tr:hover {
.add-account-form {
display: grid;
gap: 8px;
margin-top: 8px;
padding: 12px;
gap: 10px;
margin-top: 10px;
padding: 14px;
border: 1px solid var(--line);
border-radius: 16px;
background: #0f0f0f;
border-radius: 18px;
background: #101010;
}
.import-account-row {
display: grid;
gap: 8px;
margin-top: 12px;
padding: 14px;
border: 1px solid var(--line);
border-radius: 18px;
background: #101010;
}
/* ── Active Account Card ─────────────────────────────── */
@@ -1064,7 +1275,7 @@ tbody tr:hover {
}
.viewer-sidebar.open::before {
content: "";
content: '';
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.5);
+2 -2
View File
@@ -1,10 +1,10 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper API</title>
<link rel="stylesheet" href="/static/style.css" />
<link rel="stylesheet" href="/static/style.css?v=6" />
</head>
<body>
<div class="viewer-shell">
+15 -15
View File
@@ -1,37 +1,37 @@
async function loadSpec() {
const response = await fetch("/openapi.json");
const response = await fetch('/openapi.json');
const spec = await response.json();
if (!response.ok) {
throw new Error(spec.error || "Failed to load OpenAPI spec");
throw new Error(spec.error || 'Failed to load OpenAPI spec');
}
return spec;
}
function methodPayload(operation) {
const body = operation.requestBody?.content?.["application/json"]?.schema;
if (!body) return "";
const body = operation.requestBody?.content?.['application/json']?.schema;
if (!body) return '';
return JSON.stringify(body.example || body.properties || body, null, 2);
}
function renderSpec(spec) {
const root = document.getElementById("api-docs");
const sectionTemplate = document.getElementById("api-section-template");
const methodTemplate = document.getElementById("api-method-template");
root.innerHTML = "";
const root = document.getElementById('api-docs');
const sectionTemplate = document.getElementById('api-section-template');
const methodTemplate = document.getElementById('api-method-template');
root.innerHTML = '';
Object.entries(spec.paths || {}).forEach(([path, methods]) => {
const section = sectionTemplate.content.firstElementChild.cloneNode(true);
section.querySelector(".api-path").textContent = path;
const methodsRoot = section.querySelector(".api-methods");
section.querySelector('.api-path').textContent = path;
const methodsRoot = section.querySelector('.api-methods');
Object.entries(methods).forEach(([method, operation]) => {
const node = methodTemplate.content.firstElementChild.cloneNode(true);
node.querySelector(".api-verb").textContent = method.toUpperCase();
node.querySelector(".api-summary").textContent = operation.summary || "";
node.querySelector(".api-description").textContent = operation.description || "";
node.querySelector('.api-verb').textContent = method.toUpperCase();
node.querySelector('.api-summary').textContent = operation.summary || '';
node.querySelector('.api-description').textContent = operation.description || '';
const body = methodPayload(operation);
const bodyNode = node.querySelector(".api-body");
const bodyNode = node.querySelector('.api-body');
if (body) {
bodyNode.textContent = body;
} else {
@@ -49,5 +49,5 @@ loadSpec()
.then(renderSpec)
.catch((error) => {
console.error(error);
document.getElementById("api-docs").innerHTML = `<section class="panel">${error.message}</section>`;
document.getElementById('api-docs').innerHTML = `<section class="panel">${error.message}</section>`;
});
+14 -3
View File
@@ -1,10 +1,10 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Viewer</title>
<link rel="stylesheet" href="/static/style.css?v=4" />
<link rel="stylesheet" href="/static/style.css?v=6" />
</head>
<body>
<div class="viewer-shell">
@@ -13,7 +13,10 @@
<div>
<div class="eyebrow">Export Viewer</div>
<h1>Messages</h1>
<p class="muted small">Account: <select id="viewer-account-select"></select></p>
<p class="muted small">
Account:
<select id="viewer-account-select"></select>
</p>
</div>
<button id="sidebar-toggle" class="button button-small sidebar-toggle"></button>
<a class="button button-small" href="/">Dashboard</a>
@@ -27,6 +30,14 @@
<h2 id="viewer-title">Select a channel</h2>
<p id="viewer-subtitle" class="muted">Reading messages from the local SQLite database.</p>
</div>
<div class="viewer-tools">
<input id="viewer-search" type="search" placeholder="Search messages" />
<label class="viewer-auto-refresh">
<input id="viewer-auto-refresh" type="checkbox" checked />
<span>Auto-refresh</span>
</label>
<button id="viewer-refresh-btn" class="button button-small" type="button">Refresh</button>
</div>
</header>
<section id="messages-list" class="messages-list">
+217 -129
View File
@@ -1,4 +1,4 @@
if ("scrollRestoration" in history) history.scrollRestoration = "manual";
if ('scrollRestoration' in history) history.scrollRestoration = 'manual';
const viewerState = {
accountId: null,
@@ -8,7 +8,9 @@ const viewerState = {
oldestMessageId: null,
newestMessageId: null,
userId: null,
search: '',
loading: false,
autoRefreshTimer: null,
sentinelObserver: null,
};
@@ -16,98 +18,139 @@ async function api(path) {
const response = await fetch(path);
const data = await response.json();
if (!response.ok) {
throw new Error(data.error || "Request failed");
throw new Error(data.error || 'Request failed');
}
return data;
}
function showToast(message, type = 'info') {
let root = document.getElementById('toast-root');
if (!root) {
root = document.createElement('div');
root.id = 'toast-root';
root.className = 'toast-root';
document.body.appendChild(root);
}
const toast = document.createElement('div');
toast.className = `toast toast-${type}`;
toast.textContent = message;
root.appendChild(toast);
window.setTimeout(() => {
toast.classList.add('toast-hide');
window.setTimeout(() => toast.remove(), 250);
}, 3600);
}
function formatDateHeader(dateStr) {
if (!dateStr) return "";
const d = new Date(dateStr.replace(" ", "T"));
if (isNaN(d.getTime())) return "";
if (!dateStr) return '';
const d = new Date(dateStr.replace(' ', 'T'));
if (isNaN(d.getTime())) return '';
const now = new Date();
const diff = now - d;
const oneDay = 86400000;
if (diff < oneDay && d.getDate() === now.getDate()) return "Today";
if (diff < 2 * oneDay && d.getDate() === now.getDate() - 1) return "Yesterday";
return d.toLocaleDateString("en-US", {
month: "long",
day: "numeric",
year: d.getFullYear() !== now.getFullYear() ? "numeric" : undefined,
if (diff < oneDay && d.getDate() === now.getDate()) return 'Today';
if (diff < 2 * oneDay && d.getDate() === now.getDate() - 1) return 'Yesterday';
return d.toLocaleDateString('en-US', {
month: 'long',
day: 'numeric',
year: d.getFullYear() !== now.getFullYear() ? 'numeric' : undefined,
});
}
function formatTime(dateStr) {
if (!dateStr) return "";
const d = new Date(dateStr.replace(" ", "T"));
if (isNaN(d.getTime())) return "";
return d.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit" });
if (!dateStr) return '';
const d = new Date(dateStr.replace(' ', 'T'));
if (isNaN(d.getTime())) return '';
return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
}
function chatListTime(dateStr) {
if (!dateStr) return "";
const d = new Date(dateStr.replace(" ", "T"));
if (isNaN(d.getTime())) return "";
if (!dateStr) return '';
const d = new Date(dateStr.replace(' ', 'T'));
if (isNaN(d.getTime())) return '';
const now = new Date();
if (d.toDateString() === now.toDateString()) {
return d.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit" });
return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
}
return d.toLocaleDateString("en-US", { month: "short", day: "numeric" });
return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
}
function initials(value) {
const source = String(value || "?").replace(/^@/, "").trim();
const source = String(value || '?')
.replace(/^@/, '')
.trim();
const words = source.split(/\s+/).filter(Boolean);
if (words.length > 1) return (words[0][0] + words[1][0]).toUpperCase();
return source.slice(0, 2).toUpperCase();
}
function dateKey(dateStr) {
if (!dateStr) return "";
if (!dateStr) return '';
return dateStr.slice(0, 10);
}
function textNodeWithBreaks(text) {
const fragment = document.createDocumentFragment();
if (!text) return fragment;
const parts = text.split("\n");
const parts = text.split('\n');
parts.forEach((part, index) => {
if (index > 0) fragment.appendChild(document.createElement("br"));
if (index > 0) fragment.appendChild(document.createElement('br'));
fragment.appendChild(document.createTextNode(part));
});
return fragment;
}
function scrollToBottom() {
const list = document.getElementById("messages-list");
const list = document.getElementById('messages-list');
if (!list) return;
const go = () => { list.scrollTop = list.scrollHeight; };
const go = () => {
list.scrollTop = list.scrollHeight;
};
go();
requestAnimationFrame(go);
setTimeout(go, 150);
}
function makeScrollSentinel() {
const sentinel = document.createElement('div');
sentinel.id = 'scroll-sentinel';
sentinel.className = 'scroll-sentinel';
return sentinel;
}
function resetMessageView(title = 'Select a channel', subtitle = 'Reading messages from the local SQLite database.') {
const root = document.getElementById('messages-list');
if (root) {
root.innerHTML = '';
root.appendChild(makeScrollSentinel());
}
document.getElementById('viewer-title').textContent = title;
document.getElementById('viewer-subtitle').textContent = subtitle;
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
}
function renderChannelList() {
const root = document.getElementById("viewer-channel-list");
const root = document.getElementById('viewer-channel-list');
if (!root) return;
root.innerHTML = "";
const template = document.getElementById("viewer-channel-template");
root.innerHTML = '';
const template = document.getElementById('viewer-channel-template');
if (!template) return;
viewerState.channels.forEach((channel) => {
const node = template.content.firstElementChild.cloneNode(true);
const preview =
channel.last_message_preview ||
(channel.has_database ? "No text in the last saved message" : "No local database yet");
node.querySelector(".viewer-channel-avatar").textContent = initials(channel.name || channel.channel_id);
node.querySelector(".viewer-channel-name").textContent = channel.name;
node.querySelector(".viewer-channel-time").textContent = chatListTime(channel.last_date);
node.querySelector(".viewer-channel-preview").textContent = preview;
node.querySelector(".viewer-channel-count").textContent = String(channel.message_count || 0);
(channel.has_database ? 'No text in the last saved message' : 'No local database yet');
node.querySelector('.viewer-channel-avatar').textContent = initials(channel.name || channel.channel_id);
node.querySelector('.viewer-channel-name').textContent = channel.name;
node.querySelector('.viewer-channel-time').textContent = chatListTime(channel.last_date);
node.querySelector('.viewer-channel-preview').textContent = preview;
node.querySelector('.viewer-channel-count').textContent = String(channel.message_count || 0);
if (channel.channel_id === viewerState.channelId) {
node.classList.add("active");
node.classList.add('active');
}
node.addEventListener("click", () => {
document.querySelector(".viewer-sidebar")?.classList.remove("open");
node.addEventListener('click', () => {
document.querySelector('.viewer-sidebar')?.classList.remove('open');
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
loadChannel(channel.channel_id);
@@ -116,33 +159,34 @@ function renderChannelList() {
});
}
function messageEndpoint(channelId, before = "") {
function messageEndpoint(channelId, before = '') {
const search = viewerState.search ? `&search=${encodeURIComponent(viewerState.search)}` : '';
if (viewerState.accountId) {
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`;
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
}
return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`;
return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
}
function channelsEndpoint() {
if (viewerState.accountId) {
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels`;
}
return "/api/channels";
return '/api/channels';
}
function renderAccountSelector() {
const sel = document.getElementById("viewer-account-select");
const sel = document.getElementById('viewer-account-select');
if (!sel) return;
sel.innerHTML = "";
sel.innerHTML = '';
const legacy = viewerState.accounts.length === 0;
if (legacy) {
const opt = document.createElement("option");
opt.value = "";
opt.textContent = "Legacy";
const opt = document.createElement('option');
opt.value = '';
opt.textContent = 'Legacy';
sel.appendChild(opt);
} else {
viewerState.accounts.forEach((acc) => {
const opt = document.createElement("option");
const opt = document.createElement('option');
opt.value = acc.id;
opt.textContent = acc.label || acc.id;
if (acc.id === viewerState.accountId) opt.selected = true;
@@ -153,9 +197,12 @@ function renderAccountSelector() {
async function switchViewerAccount(accountId) {
viewerState.accountId = accountId || null;
viewerState.search = '';
const searchInput = document.getElementById('viewer-search');
if (searchInput) searchInput.value = '';
viewerState.accounts.forEach((acc) => {
if (acc.id === accountId) {
const opts = document.getElementById("viewer-account-select")?.options;
const opts = document.getElementById('viewer-account-select')?.options;
if (opts) {
for (let i = 0; i < opts.length; i++) {
opts[i].selected = opts[i].value === accountId;
@@ -166,10 +213,10 @@ async function switchViewerAccount(accountId) {
try {
const authData = viewerState.accountId
? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`)
: await api("/api/auth");
: await api('/api/auth');
viewerState.userId = authData.user_id || null;
} catch (e) {
console.warn("Could not fetch user ID:", e);
console.warn('Could not fetch user ID:', e);
}
const channels = await api(channelsEndpoint());
viewerState.channels = channels;
@@ -179,94 +226,95 @@ async function switchViewerAccount(accountId) {
renderChannelList();
if (viewerState.channelId) {
await loadChannel(viewerState.channelId);
} else {
resetMessageView(
'No channels',
viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.',
);
}
}
async function loadViewerAccount(params) {
const requested = params.get("account");
const requested = params.get('account');
try {
const payload = await api("/api/accounts");
const payload = await api('/api/accounts');
viewerState.accounts = payload.accounts || [];
} catch {
viewerState.accounts = [];
}
viewerState.accountId =
requested ||
viewerState.accounts[0]?.id ||
null;
const requestedExists = requested && viewerState.accounts.some((acc) => acc.id === requested);
viewerState.accountId = requestedExists ? requested : viewerState.accounts[0]?.id || null;
renderAccountSelector();
const sel = document.getElementById("viewer-account-select");
const sel = document.getElementById('viewer-account-select');
if (sel) {
sel.addEventListener("change", () => switchViewerAccount(sel.value));
sel.addEventListener('change', () => switchViewerAccount(sel.value));
}
try {
const authData = viewerState.accountId
? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`)
: await api("/api/auth");
: await api('/api/auth');
viewerState.userId = authData.user_id || null;
} catch (e) {
console.warn("Could not fetch user ID:", e);
console.warn('Could not fetch user ID:', e);
}
}
function buildMessageNode(message, root) {
const template = document.getElementById("message-template");
const template = document.getElementById('message-template');
const wrap = template.content.firstElementChild.cloneNode(true);
const bubble = wrap.querySelector(".chat-bubble");
const bubble = wrap.querySelector('.chat-bubble');
bubble.dataset.messageId = String(message.message_id);
const isOwn = viewerState.userId && message.sender_id === viewerState.userId;
if (isOwn) {
wrap.classList.add("chat-bubble-wrap--own");
wrap.classList.add('chat-bubble-wrap--own');
}
const sender = bubble.querySelector(".chat-sender");
sender.textContent = message.sender_name || "";
const sender = bubble.querySelector('.chat-sender');
sender.textContent = message.sender_name || '';
const replyEl = bubble.querySelector(".chat-reply");
const replyEl = bubble.querySelector('.chat-reply');
const replyMsg = message.reply_to_message;
if (message.reply_to) {
replyEl.classList.remove("hidden");
bubble.querySelector(".chat-reply-author").textContent =
replyMsg?.sender_name || `#${message.reply_to}`;
bubble.querySelector(".chat-reply-text").textContent =
replyMsg?.text || "(message not available)";
replyEl.addEventListener("click", () => {
replyEl.classList.remove('hidden');
bubble.querySelector('.chat-reply-author').textContent = replyMsg?.sender_name || `#${message.reply_to}`;
bubble.querySelector('.chat-reply-text').textContent = replyMsg?.text || '(message not available)';
replyEl.addEventListener('click', () => {
const target = root.querySelector(`[data-message-id="${message.reply_to}"]`);
if (target) {
target.scrollIntoView({ behavior: "smooth", block: "center" });
target.classList.add("chat-bubble--highlight");
setTimeout(() => target.classList.remove("chat-bubble--highlight"), 2000);
target.scrollIntoView({ behavior: 'smooth', block: 'center' });
target.classList.add('chat-bubble--highlight');
setTimeout(() => target.classList.remove('chat-bubble--highlight'), 2000);
}
});
} else {
replyEl.classList.add("hidden");
replyEl.classList.add('hidden');
}
bubble.querySelector(".chat-text").appendChild(textNodeWithBreaks(message.text || ""));
bubble.querySelector('.chat-text').appendChild(textNodeWithBreaks(message.text || ''));
const media = bubble.querySelector(".chat-media");
if (message.media_url && message.media_kind === "image") {
const img = document.createElement("img");
const media = bubble.querySelector('.chat-media');
if (message.media_url && message.media_kind === 'image') {
const img = document.createElement('img');
img.src = message.media_url;
img.loading = "lazy";
img.loading = 'lazy';
media.appendChild(img);
} else if (message.media_url && message.media_kind === "video") {
const video = document.createElement("video");
} else if (message.media_url && message.media_kind === 'video') {
const video = document.createElement('video');
video.src = message.media_url;
video.controls = true;
video.preload = "metadata";
video.preload = 'metadata';
media.appendChild(video);
} else if (message.media_url) {
const link = document.createElement("a");
const link = document.createElement('a');
link.href = message.media_url;
link.target = "_blank";
link.rel = "noreferrer";
link.textContent = "Open file";
link.target = '_blank';
link.rel = 'noreferrer';
link.textContent = 'Open file';
media.appendChild(link);
}
@@ -275,40 +323,40 @@ function buildMessageNode(message, root) {
if (message.views) footerParts.push(`views: ${message.views}`);
if (message.forwards) footerParts.push(`forwards: ${message.forwards}`);
if (message.reactions) footerParts.push(`reactions: ${message.reactions}`);
bubble.querySelector(".chat-footer").textContent = footerParts.join(" | ");
bubble.querySelector('.chat-footer').textContent = footerParts.join(' | ');
return { wrap, dateKey: dateKey(message.date) };
}
function makeDateSep(text) {
const el = document.createElement("div");
el.className = "chat-date-sep";
const el = document.createElement('div');
el.className = 'chat-date-sep';
el.textContent = text;
return el;
}
function renderMessages(payload, append = false) {
const root = document.getElementById("messages-list");
const sentinel = document.getElementById("scroll-sentinel");
const root = document.getElementById('messages-list');
if (!root) return;
let sentinel = document.getElementById('scroll-sentinel');
if (!sentinel) sentinel = makeScrollSentinel();
const channel = payload.channel;
document.getElementById("viewer-title").textContent =
channel?.name || payload.channel_id;
document.getElementById("viewer-subtitle").textContent = channel
? `${channel.message_count} messages, latest: ${channel.last_date || "-"}`
: "No local database found for this channel yet.";
document.getElementById('viewer-title').textContent = channel?.name || payload.channel_id;
document.getElementById('viewer-subtitle').textContent = channel
? `${payload.messages.length} shown${viewerState.search ? ` for "${viewerState.search}"` : ''}, ${channel.message_count} total, latest: ${channel.last_date || '-'}`
: 'No local database found for this channel yet.';
if (!append) {
root.innerHTML = "";
root.innerHTML = '';
root.appendChild(sentinel);
}
let lastKey = null;
if (append) {
const firstExisting = root.querySelector("[data-message-id]");
const firstExisting = root.querySelector('[data-message-id]');
if (firstExisting) {
const candidate = firstExisting.closest(".chat-bubble-wrap");
const candidate = firstExisting.closest('.chat-bubble-wrap');
if (candidate && candidate._dateKey) lastKey = candidate._dateKey;
}
}
@@ -342,8 +390,16 @@ function renderMessages(payload, append = false) {
if (payload.messages.length) {
viewerState.oldestMessageId = payload.messages[0].message_id;
viewerState.newestMessageId =
payload.messages[payload.messages.length - 1].message_id;
viewerState.newestMessageId = payload.messages[payload.messages.length - 1].message_id;
} else if (!append) {
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
const empty = document.createElement('div');
empty.className = 'viewer-empty-state';
empty.textContent = viewerState.search
? `No messages found for "${viewerState.search}".`
: 'No saved messages in this channel yet.';
root.appendChild(empty);
}
if (!append) {
@@ -357,14 +413,11 @@ async function loadChannel(channelId, append = false) {
viewerState.channelId = channelId;
renderChannelList();
const before =
append && viewerState.oldestMessageId
? `&before=${viewerState.oldestMessageId}`
: "";
const before = append && viewerState.oldestMessageId ? `&before=${viewerState.oldestMessageId}` : '';
try {
const payload = await api(messageEndpoint(channelId, before));
if (append) {
const list = document.getElementById("messages-list");
const list = document.getElementById('messages-list');
const prevScrollHeight = list.scrollHeight;
renderMessages(payload, true);
requestAnimationFrame(() => {
@@ -374,14 +427,48 @@ async function loadChannel(channelId, append = false) {
renderMessages(payload, false);
}
} catch (err) {
console.error("Failed to load messages:", err);
console.error('Failed to load messages:', err);
} finally {
viewerState.loading = false;
}
}
async function refreshCurrentChannel() {
if (!viewerState.channelId || viewerState.loading) return;
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
await loadChannel(viewerState.channelId);
}
function setupViewerTools() {
const searchInput = document.getElementById('viewer-search');
const refreshBtn = document.getElementById('viewer-refresh-btn');
const autoRefresh = document.getElementById('viewer-auto-refresh');
let searchTimer = null;
if (searchInput) {
searchInput.addEventListener('input', () => {
clearTimeout(searchTimer);
searchTimer = setTimeout(() => {
viewerState.search = searchInput.value.trim();
refreshCurrentChannel();
}, 250);
});
}
if (refreshBtn) {
refreshBtn.addEventListener('click', () => refreshCurrentChannel());
}
viewerState.autoRefreshTimer = window.setInterval(() => {
if (!autoRefresh || autoRefresh.checked) {
refreshCurrentChannel();
}
}, 30000);
}
function setupInfiniteScroll() {
const sentinel = document.getElementById("scroll-sentinel");
const sentinel = document.getElementById('scroll-sentinel');
if (!sentinel) return;
if (viewerState.sentinelObserver) {
viewerState.sentinelObserver.disconnect();
@@ -389,17 +476,12 @@ function setupInfiniteScroll() {
viewerState.sentinelObserver = new IntersectionObserver(
(entries) => {
for (const entry of entries) {
if (
entry.isIntersecting &&
viewerState.channelId &&
viewerState.oldestMessageId &&
!viewerState.loading
) {
if (entry.isIntersecting && viewerState.channelId && viewerState.oldestMessageId && !viewerState.loading) {
loadChannel(viewerState.channelId, true);
}
}
},
{ root: document.getElementById("messages-list"), threshold: 0.1 }
{ root: document.getElementById('messages-list'), threshold: 0.1 },
);
viewerState.sentinelObserver.observe(sentinel);
}
@@ -408,27 +490,33 @@ async function main() {
const params = new URLSearchParams(window.location.search);
await loadViewerAccount(params);
viewerState.channels = await api(channelsEndpoint());
viewerState.channelId =
params.get("channel") ||
viewerState.channels[0]?.channel_id ||
null;
const requestedChannel = params.get('channel');
const requestedChannelExists =
requestedChannel && viewerState.channels.some((channel) => channel.channel_id === requestedChannel);
viewerState.channelId = requestedChannelExists ? requestedChannel : viewerState.channels[0]?.channel_id || null;
renderChannelList();
if (viewerState.channelId) {
await loadChannel(viewerState.channelId);
} else {
resetMessageView(
'No channels',
viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.',
);
}
setupInfiniteScroll();
setupViewerTools();
const toggle = document.getElementById("sidebar-toggle");
const sidebar = document.querySelector(".viewer-sidebar");
const toggle = document.getElementById('sidebar-toggle');
const sidebar = document.querySelector('.viewer-sidebar');
if (toggle && sidebar) {
toggle.addEventListener("click", () => {
sidebar.classList.toggle("open");
toggle.addEventListener('click', () => {
sidebar.classList.toggle('open');
});
sidebar.addEventListener("click", (e) => {
sidebar.addEventListener('click', (e) => {
if (e.target === sidebar) {
sidebar.classList.remove("open");
sidebar.classList.remove('open');
}
});
}
@@ -436,5 +524,5 @@ async function main() {
main().catch((error) => {
console.error(error);
alert(error.message);
showToast(error.message, 'error');
});
+297 -15
View File
@@ -194,6 +194,7 @@ def load_messages(
channel_id: str,
limit: int = 120,
before_message_id: Optional[int] = None,
search: Optional[str] = None,
) -> List[Dict[str, Any]]:
db_path = channel_db_path(account_id, channel_id)
if not db_path.exists():
@@ -207,9 +208,15 @@ def load_messages(
"FROM messages "
)
params: List[Any] = []
where: List[str] = []
if before_message_id is not None:
query += "WHERE message_id < ? "
where.append("message_id < ?")
params.append(before_message_id)
if search:
where.append("message LIKE ?")
params.append(f"%{search}%")
if where:
query += "WHERE " + " AND ".join(where) + " "
query += "ORDER BY message_id DESC LIMIT ?"
params.append(limit)
rows = conn.execute(query, params).fetchall()
@@ -323,8 +330,22 @@ class JobRunner:
def create_job(self, job_type: str, title: str, payload: Dict[str, Any]) -> Job:
if self._shutdown_flag:
raise RuntimeError("Server is shutting down, cannot create new jobs")
job_id = f"job-{int(time.time() * 1000)}"
account_id = payload.get("account_id")
with self.lock:
if account_id:
for job_id in self.job_order:
existing = self.jobs[job_id]
if (
existing.account_id == account_id
and existing.status in {"queued", "running"}
):
existing.logs = (
existing.logs
+ "\n"
+ f"[{datetime.now().strftime('%H:%M:%S')}] Reused existing active job for this account."
).strip()
return existing
job_id = f"job-{int(time.time() * 1000)}"
job = Job(
job_id=job_id,
job_type=job_type,
@@ -332,7 +353,6 @@ class JobRunner:
payload=payload,
account_id=account_id,
)
with self.lock:
self.jobs[job_id] = job
self.job_order.insert(0, job_id)
self.job_order = self.job_order[:20]
@@ -346,6 +366,15 @@ class JobRunner:
result = [j for j in result if j.get("account_id") == account_id]
return result
def active_jobs_by_account(self) -> Dict[str, Dict[str, Any]]:
with self.lock:
active = {}
for job_id in self.job_order:
job = self.jobs[job_id]
if job.account_id and job.status in {"queued", "running"}:
active[job.account_id] = job.to_dict()
return active
def get_job(self, job_id: str) -> Optional[Dict[str, Any]]:
with self.lock:
job = self.jobs.get(job_id)
@@ -731,6 +760,33 @@ class PerAccountContinuousScrapeManager:
}
store.update(mutate)
def refresh_config(self) -> None:
"""
Re-read account state from disk and sync the in-memory config.
Call this after external state changes (import, channel add/remove)
so the manager picks up new channels or config updates without a restart.
"""
disk_state = load_account(DATA_DIR, self.account_id)
disk_cfg = disk_state.get("continuous_scraping") or {}
with self.lock:
# Merge disk config into memory, preserving our current running state
self.config["enabled"] = bool(disk_cfg.get("enabled", False))
self.config["interval_minutes"] = max(1, int(disk_cfg.get("interval_minutes", 1) or 1))
self.config["channels"] = [
str(item).strip()
for item in disk_cfg.get("channels", [])
if str(item).strip()
]
self.config["run_all_tracked"] = bool(disk_cfg.get("run_all_tracked", True))
# Sync running state with desired enabled state
if self.config["enabled"] and not self.status["running"]:
pass # don't auto-start — user must call start()
elif not self.config["enabled"] and self.status["running"]:
self._log("Continuous disabled via external state change, stopping.", "warn")
self.stop_event.set()
self.status["running"] = False
self._log("Config refreshed from disk.", "debug")
def _log(self, message: str, level: str = "debug") -> None:
timestamp = utc_now_iso()
entry = {"timestamp": timestamp, "level": level, "message": message}
@@ -811,6 +867,23 @@ class PerAccountContinuousScrapeManager:
def _run_loop(self) -> None:
while not self.stop_event.is_set():
# Refresh config from disk so channel / setting changes take effect
self.refresh_config()
# Check auth — don't iterate if account isn't authorized
auth_info = auth_status_for(self.account_id)
if auth_info.get("status") not in ("ready", "authorized"):
self._log(
f"Account not authorized (status={auth_info.get('status')}), "
"skipping iteration",
"warn",
)
sleep_seconds = max(5, 60)
interrupted = self.stop_event.wait(timeout=sleep_seconds)
if interrupted:
break
continue
channels = self._resolve_channels()
cfg = self.snapshot()["config"]
interval_minutes = cfg.get("interval_minutes", 1)
@@ -893,6 +966,11 @@ class ContinuousScrapeOrchestrator:
result[account_id] = dict(mgr.snapshot())
return result
def refresh_account(self, account_id: str) -> None:
"""Re-read account state from disk and sync the continuous manager config."""
mgr = self._get_or_create(account_id)
mgr.refresh_config()
def update_for(
self,
account_id: str,
@@ -902,12 +980,14 @@ class ContinuousScrapeOrchestrator:
run_all_tracked: bool,
) -> Dict[str, Any]:
mgr = self._get_or_create(account_id)
return mgr.update(enabled, interval_minutes, channels, run_all_tracked)
result = mgr.update(enabled, interval_minutes, channels, run_all_tracked)
# After explicit update, sync state to disk is already done by mgr.update
return result
def add_account(self, account_id: str) -> None:
def add_account(self, account_id: str, auto_start: bool = True) -> None:
acc_state = load_account(DATA_DIR, account_id)
cfg = acc_state.get("continuous_scraping", {})
if cfg.get("enabled", True) and START_CONTINUOUS:
if auto_start and cfg.get("enabled", False) and START_CONTINUOUS:
self.start_account(account_id)
def remove_account(self, account_id: str) -> None:
@@ -924,7 +1004,7 @@ class ContinuousScrapeOrchestrator:
for account_id in list_accounts(DATA_DIR):
acc_state = load_account(DATA_DIR, account_id)
cfg = acc_state.get("continuous_scraping", {})
if cfg.get("enabled", True):
if cfg.get("enabled", False) and START_CONTINUOUS:
self.start_account(account_id)
@@ -973,6 +1053,27 @@ def auth_status_for(account_id: Optional[str] = None) -> Dict[str, Any]:
return status
def account_health_summary(account_id: str, job_runner: Optional[JobRunner] = None) -> Dict[str, Any]:
acc_state = load_account(DATA_DIR, account_id)
acc_dir = account_data_dir(DATA_DIR, account_id)
session_file = Path(account_session_path(SESSION_DIR, account_id))
channels = list_channels_snapshot(account_id)
active_job = (job_runner.active_jobs_by_account().get(account_id) if job_runner else None)
last_scrape = next((item.get("last_date") for item in channels if item.get("last_date")), None)
return {
"account_id": account_id,
"label": acc_state.get("label", ""),
"data_dir_exists": acc_dir.exists(),
"session_ready": session_file.exists(),
"api_credentials": bool(acc_state.get("api_id") and acc_state.get("api_hash")),
"channel_count": len(channels),
"message_count": sum(int(item.get("message_count") or 0) for item in channels),
"media_count": sum(int(item.get("media_count") or 0) for item in channels),
"last_scrape": last_scrape,
"active_job": active_job,
}
def auth_status() -> Dict[str, Any]:
return auth_status_for(account_id=None)
@@ -1166,6 +1267,12 @@ def openapi_payload() -> Dict[str, Any]:
"in": "query",
"schema": {"type": "integer"},
},
{
"name": "search",
"in": "query",
"schema": {"type": "string"},
"description": "Full-text search in message text",
},
],
"responses": json_response,
}
@@ -1229,6 +1336,31 @@ def openapi_payload() -> Dict[str, Any]:
},
}
},
"/api/jobs/{job_id}/events": {
"get": {
"summary": "Job SSE event stream",
"description": "Server-Sent Events stream of job status updates. Closes when job completes or fails.",
"parameters": [
{
"name": "job_id",
"in": "path",
"required": True,
"schema": {"type": "string"},
}
],
"responses": {
"200": {
"description": "SSE stream of job status objects",
"content": {
"text/event-stream": {
"schema": {"type": "string"}
}
},
},
"404": {"description": "Job not found"},
},
}
},
"/api/jobs/scrape": {
"post": {
"summary": "Scrape one channel or all tracked channels",
@@ -1318,6 +1450,23 @@ def openapi_payload() -> Dict[str, Any]:
"responses": {**json_response, **error_response},
},
},
"/api/accounts/import": {
"post": {
"summary": "Import account from exported JSON",
"requestBody": json_body(
{
"account_id": {"type": "string", "example": "work"},
"state": {
"type": "object",
"description": "Exported account state object",
"example": {"label": "Work", "api_id": 123456, "api_hash": "abc", "channels": {}, "continuous_scraping": {"enabled": False}},
},
},
["account_id", "state"],
),
"responses": {**json_response, **error_response},
}
},
"/api/accounts/{id}": {
"get": {
"summary": "Account dashboard",
@@ -1344,6 +1493,34 @@ def openapi_payload() -> Dict[str, Any]:
"responses": json_response,
},
},
"/api/accounts/{id}/export": {
"get": {
"summary": "Export account settings as JSON",
"parameters": [
{
"name": "id",
"in": "path",
"required": True,
"schema": {"type": "string"},
}
],
"responses": json_response,
}
},
"/api/accounts/{id}/health": {
"get": {
"summary": "Account health summary",
"parameters": [
{
"name": "id",
"in": "path",
"required": True,
"schema": {"type": "string"},
}
],
"responses": json_response,
}
},
"/api/accounts/{id}/auth": {
"get": {
"summary": "Account auth snapshot",
@@ -1452,6 +1629,12 @@ def openapi_payload() -> Dict[str, Any]:
"in": "query",
"schema": {"type": "integer"},
},
{
"name": "search",
"in": "query",
"schema": {"type": "string"},
"description": "Full-text search in message text",
},
],
"responses": json_response,
}
@@ -1594,6 +1777,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/":
return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8")
if path == "/settings":
return self.serve_file(WEBUI_DIR / "settings.html", "text/html; charset=utf-8")
if path == "/viewer":
return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8")
if path in {"/swagger", "/swigger", "/docs"}:
@@ -1634,6 +1819,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
)
if path == "/api/jobs":
return self.send_json(self.app.job_runner.recent_jobs())
if path.startswith("/api/jobs/") and path.endswith("/events"):
job_id = path.split("/")[-2]
return self.stream_job_events(job_id)
if path.startswith("/api/jobs/"):
job_id = path.rsplit("/", 1)[-1]
job = self.app.job_runner.get_job(job_id)
@@ -1644,14 +1832,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_json(list_channels_snapshot())
if path.startswith("/api/channels/") and path.endswith("/messages"):
parts = path.split("/")
channel_id = parts[3]
channel_id = urllib.parse.unquote(parts[3]).lstrip("@")
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
before = query.get("before")
before_message_id = int(before[0]) if before else None
search = (query.get("search") or query.get("q") or [""])[0].strip()
payload = {
"channel_id": channel_id,
"messages": load_messages(
None, channel_id, limit=limit, before_message_id=before_message_id
None, channel_id, limit=limit, before_message_id=before_message_id,
search=search or None
),
"channel": next(
(
@@ -1674,7 +1864,6 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
def _legacy_auth_snapshot(self) -> Dict[str, Any]:
state = load_state()
session_ready = (SESSION_DIR / "session.session").exists()
return {
"phase": "unknown",
"status": "unknown",
@@ -1703,6 +1892,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
"auth": auth_info,
"status": auth_info.get("status", "unknown"),
"continuous_running": cs_info.get("status", {}).get("running", False),
"health": account_health_summary(account_id, self.app.job_runner),
})
return self.send_json({"accounts": accounts})
@@ -1719,10 +1909,19 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self._handle_get_account_dashboard(account_id)
if sub == ["auth"]:
return self.send_json(self.app.auth_manager.auth_state(account_id))
if sub == ["health"]:
return self.send_json(account_health_summary(account_id, self.app.job_runner))
if sub == ["export"]:
payload = {
"version": 1,
"account_id": account_id,
"state": load_account(DATA_DIR, account_id),
}
return self.send_json(payload)
if sub == ["channels"]:
return self._handle_get_account_channels(account_id)
if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages":
channel_id = sub[1]
channel_id = urllib.parse.unquote(sub[1]).lstrip("@")
return self._handle_get_account_channel_messages(account_id, channel_id, query)
if sub == ["continuous"]:
return self.send_json(
@@ -1744,9 +1943,37 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
"jobs": self.app.job_runner.recent_jobs(account_id=account_id),
"continuous": cs_info,
"scrape_media": bool(acc_state.get("scrape_media", True)),
"health": account_health_summary(account_id, self.app.job_runner),
}
return self.send_json(payload)
def stream_job_events(self, job_id: str) -> None:
if not self.app.job_runner.get_job(job_id):
return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found")
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "keep-alive")
self.end_headers()
last_payload = None
deadline = time.time() + 60 * 30
while time.time() < deadline:
job = self.app.job_runner.get_job(job_id)
if not job:
break
payload = json.dumps(job, ensure_ascii=False)
if payload != last_payload:
try:
self.wfile.write(f"data: {payload}\n\n".encode("utf-8"))
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, OSError):
break
last_payload = payload
if job.get("status") in {"completed", "failed"}:
break
time.sleep(1)
def _handle_get_account_channels(self, account_id: str) -> None:
return self.send_json(list_channels_snapshot(account_id))
@@ -1756,10 +1983,12 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
before = query.get("before")
before_message_id = int(before[0]) if before else None
search = (query.get("search") or query.get("q") or [""])[0].strip()
payload = {
"channel_id": channel_id,
"messages": load_messages(
account_id, channel_id, limit=limit, before_message_id=before_message_id
, search=search or None
),
"channel": next(
(
@@ -1780,6 +2009,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if path == "/":
return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8", head_only=True)
if path == "/settings":
return self.serve_file(WEBUI_DIR / "settings.html", "text/html; charset=utf-8", head_only=True)
if path == "/viewer":
return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8", head_only=True)
if path in {"/swagger", "/swigger", "/docs"}:
@@ -1997,6 +2228,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
# ── /api/accounts/* POST routes ─────────────────────────────────
if path == "/api/accounts":
return self._handle_post_accounts_create(body)
if path == "/api/accounts/import":
return self._handle_post_accounts_import(body)
if path.startswith("/api/accounts/"):
return self._handle_post_account(path, body)
@@ -2034,14 +2267,53 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
"scrape_media": True,
"forwarding_rules": [],
"continuous_scraping": {
"enabled": True,
"enabled": False,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
},
}
store.save(init_state)
self.app.continuous_orchestrator.add_account(account_id)
self.app.continuous_orchestrator.add_account(account_id, auto_start=False)
return self.send_json({"ok": True, "account_id": account_id})
def _handle_post_accounts_import(self, body: Dict[str, Any]) -> None:
account_id = str(body.get("account_id") or body.get("id") or "").strip()
state = body.get("state")
if not account_id:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id is required")
if not account_id.replace("-", "").replace("_", "").isalnum():
return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id must be alphanumeric (dashes and underscores allowed)")
if account_exists(DATA_DIR, account_id):
return self.send_error_json(HTTPStatus.BAD_REQUEST, f"Account '{account_id}' already exists")
if not isinstance(state, dict):
return self.send_error_json(HTTPStatus.BAD_REQUEST, "state object is required")
imported_state = {
"label": str(body.get("label") or state.get("label") or account_id).strip(),
"api_id": state.get("api_id"),
"api_hash": state.get("api_hash"),
"channels": state.get("channels") if isinstance(state.get("channels"), dict) else {},
"channel_names": state.get("channel_names") if isinstance(state.get("channel_names"), dict) else {},
"scrape_media": bool(state.get("scrape_media", True)),
"forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [],
"continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else {
"enabled": False,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
},
}
def mutate_global(global_state: Dict[str, Any]) -> None:
accounts = global_state.setdefault("accounts", [])
if account_id not in accounts:
accounts.append(account_id)
get_global_store(DATA_DIR).update(mutate_global)
get_account_store(DATA_DIR, account_id).save(imported_state)
self.app.continuous_orchestrator.add_account(account_id, auto_start=False)
self.app.continuous_orchestrator.refresh_account(account_id)
return self.send_json({"ok": True, "account_id": account_id})
def _handle_post_account(self, path: str, body: Dict[str, Any]) -> None:
@@ -2141,6 +2413,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
return self.send_json(payload)
def _handle_account_channel_add(self, account_id: str, channel_id: str, name: Optional[str]) -> None:
channel_id = channel_id.lstrip("@")
if not channel_id:
return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
store = get_account_store(DATA_DIR, account_id)
@@ -2150,6 +2423,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if name:
state.setdefault("channel_names", {})[channel_id] = str(name).strip()
store.update(mutate)
self.app.continuous_orchestrator.refresh_account(account_id)
return self.send_json({"ok": True, "channel_id": channel_id})
def _handle_account_channel_remove(self, account_id: str, channel_id: str) -> None:
@@ -2164,6 +2438,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
del chans[channel_id]
state.get("channel_names", {}).pop(channel_id, None)
store.update(mutate)
self.app.continuous_orchestrator.refresh_account(account_id)
return self.send_json({"ok": existed[0], "channel_id": channel_id})
def _handle_account_job_scrape(self, account_id: str, body: Dict[str, Any]) -> None:
@@ -2300,9 +2575,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
if acc_dir.exists():
import shutil
shutil.rmtree(str(acc_dir), ignore_errors=True)
if session_file.exists():
for session_sidecar in (
session_file,
Path(str(session_file) + "-wal"),
Path(str(session_file) + "-shm"),
session_file.with_suffix(session_file.suffix + "-journal"),
):
if not session_sidecar.exists():
continue
try:
session_file.unlink()
session_sidecar.unlink()
except OSError:
pass