diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml new file mode 100644 index 0000000..77c9a55 --- /dev/null +++ b/.gitea/workflows/ci.yaml @@ -0,0 +1,171 @@ +name: ci + +'on': + push: + branches: + - '**' + pull_request: + workflow_dispatch: + +env: + # Shared image coordinates for the publish job. + REGISTRY: gcr.forust.xyz + IMAGE_NAME: forust/telegram-scraper + +jobs: + lint-prettier: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check formatting with Prettier + shell: bash + run: | + mapfile -t prettier_files < <( + git ls-files \ + | grep -E '\.(md|json|ya?ml|html|css)$' \ + | grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)' + ) + + if [ "${#prettier_files[@]}" -eq 0 ]; then + echo "No Prettier-managed files found." + exit 0 + fi + + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + node:22-alpine \ + sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}" + + lint-ruff: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Lint Python with Ruff + shell: bash + run: | + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + ghcr.io/astral-sh/ruff:latest \ + check . + + lint-yaml: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Lint YAML syntax + shell: bash + run: | + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + cytopia/yamllint:latest \ + -c .yamllint . + + lint-dockerfiles: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Lint Dockerfiles + shell: bash + run: | + mapfile -t dockerfiles < <( + git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*' + ) + + if [ "${#dockerfiles[@]}" -eq 0 ]; then + echo "No Dockerfiles found." + exit 0 + fi + + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + --entrypoint hadolint \ + hadolint/hadolint:latest-debian \ + -c .hadolint.yaml "${dockerfiles[@]}" + + validate: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Validate Kubernetes manifests + shell: bash + run: | + mapfile -t manifests < <( + git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \ + | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$' + ) + + if [ "${#manifests[@]}" -eq 0 ]; then + echo "No Kubernetes manifests found." + exit 0 + fi + + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + ghcr.io/yannh/kubeconform:latest \ + -strict \ + -ignore-missing-schemas \ + -summary \ + "${manifests[@]}" + + publish: + needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate] + if: github.event_name != 'pull_request' + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Read project version + id: version + shell: bash + run: | + version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Log in to registry + shell: bash + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \ + -u "${{ secrets.REGISTRY_USERNAME }}" \ + --password-stdin + + - name: Build and push image + shell: bash + run: | + image="${REGISTRY}/${IMAGE_NAME}" + tags=("latest" "${{ steps.version.outputs.version }}") + + case "${GITHUB_REF_NAME}" in + main) + tags+=("main" "prod") + ;; + dev) + tags+=("dev") + ;; + esac + + build_args=() + for tag in "${tags[@]}"; do + build_args+=(-t "${image}:${tag}") + done + + docker build "${build_args[@]}" . + + for tag in "${tags[@]}"; do + docker push "${image}:${tag}" + done diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..77c9a55 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,171 @@ +name: ci + +'on': + push: + branches: + - '**' + pull_request: + workflow_dispatch: + +env: + # Shared image coordinates for the publish job. + REGISTRY: gcr.forust.xyz + IMAGE_NAME: forust/telegram-scraper + +jobs: + lint-prettier: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check formatting with Prettier + shell: bash + run: | + mapfile -t prettier_files < <( + git ls-files \ + | grep -E '\.(md|json|ya?ml|html|css)$' \ + | grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)' + ) + + if [ "${#prettier_files[@]}" -eq 0 ]; then + echo "No Prettier-managed files found." + exit 0 + fi + + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + node:22-alpine \ + sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}" + + lint-ruff: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Lint Python with Ruff + shell: bash + run: | + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + ghcr.io/astral-sh/ruff:latest \ + check . + + lint-yaml: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Lint YAML syntax + shell: bash + run: | + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + cytopia/yamllint:latest \ + -c .yamllint . + + lint-dockerfiles: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Lint Dockerfiles + shell: bash + run: | + mapfile -t dockerfiles < <( + git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*' + ) + + if [ "${#dockerfiles[@]}" -eq 0 ]; then + echo "No Dockerfiles found." + exit 0 + fi + + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + --entrypoint hadolint \ + hadolint/hadolint:latest-debian \ + -c .hadolint.yaml "${dockerfiles[@]}" + + validate: + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Validate Kubernetes manifests + shell: bash + run: | + mapfile -t manifests < <( + git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \ + | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$' + ) + + if [ "${#manifests[@]}" -eq 0 ]; then + echo "No Kubernetes manifests found." + exit 0 + fi + + docker run --rm \ + -v "$PWD:/work" \ + -w /work \ + ghcr.io/yannh/kubeconform:latest \ + -strict \ + -ignore-missing-schemas \ + -summary \ + "${manifests[@]}" + + publish: + needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate] + if: github.event_name != 'pull_request' + runs-on: [self-hosted, linux, arch, homelab] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Read project version + id: version + shell: bash + run: | + version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Log in to registry + shell: bash + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \ + -u "${{ secrets.REGISTRY_USERNAME }}" \ + --password-stdin + + - name: Build and push image + shell: bash + run: | + image="${REGISTRY}/${IMAGE_NAME}" + tags=("latest" "${{ steps.version.outputs.version }}") + + case "${GITHUB_REF_NAME}" in + main) + tags+=("main" "prod") + ;; + dev) + tags+=("dev") + ;; + esac + + build_args=() + for tag in "${tags[@]}"; do + build_args+=(-t "${image}:${tag}") + done + + docker build "${build_args[@]}" . + + for tag in "${tags[@]}"; do + docker push "${image}:${tag}" + done diff --git a/README.md b/README.md index 0ce0cef..98b3033 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ kubectl apply -f k8s/telegram-scraper.yaml Then open: ```text -http://:8080 +http://:7887 ``` ### Volumes (Docker) diff --git a/app_state.py b/app_state.py index 3d6a910..5a84e21 100644 --- a/app_state.py +++ b/app_state.py @@ -25,7 +25,7 @@ ACCOUNT_DEFAULTS: Dict[str, Any] = { "scrape_media": True, "forwarding_rules": [], "continuous_scraping": { - "enabled": True, + "enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True, diff --git a/compose.yaml b/compose.yaml index e36b7ba..6317346 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,13 +5,19 @@ services: dockerfile: Dockerfile container_name: telegram-scraper image: gcr.forust.xyz/forust/telegram-scraper:latest - user: 1000:1000 - restart: unless-stopped tty: true + user: '1000:1000' + restart: unless-stopped ports: - - "7887:8080" + - 7887:8080 healthcheck: - test: ["CMD", "python", "-c", "import http.client,json;c=http.client.HTTPConnection('localhost',8080);c.request('GET','/health');r=c.getresponse();exit(0)if json.loads(r.read()).get('ok')else exit(1)"] + test: + [ + CMD, + python, + -c, + "import http.client, json; c=http.client.HTTPConnection('localhost', 8080); c.request('GET', '/health'); r=c.getresponse(); exit(0) if json.loads(r.read()).get('ok') else exit(1)", + ] interval: 30s timeout: 10s retries: 3 diff --git a/health.py b/health.py index 2bdaed8..b123ecd 100644 --- a/health.py +++ b/health.py @@ -13,6 +13,14 @@ def health_payload( job_queue_size: int, account_ids: Optional[List[str]] = None, ) -> Dict[str, Any]: + """ + Return a health-check payload for the application. + + NOTE: This endpoint does NOT support message search or query parameters. + Health concerns itself with filesystem, database connectivity, job queue, + and account session state. Message search is handled by the message API + endpoints (/api/channels/*/messages, /api/accounts/*/channels/*/messages). + """ checks = { "data_dir": _dir_check(data_dir, writable=True), "session_dir": _dir_check(session_dir, writable=True), @@ -35,7 +43,13 @@ def health_payload( "path": str(session_file), }, } - checks["accounts"] = account_checks + checks["accounts"] = { + "ok": all( + item.get("data_dir", {}).get("ok", False) + for item in account_checks.values() + ), + "items": account_checks, + } ok = all(item.get("ok", False) for item in checks.values()) return {"ok": ok, "status": "ok" if ok else "degraded", "checks": checks} diff --git a/k8s/telegram-scraper.yaml b/k8s/telegram-scraper.yaml index 3bfa56d..90bbe84 100644 --- a/k8s/telegram-scraper.yaml +++ b/k8s/telegram-scraper.yaml @@ -34,8 +34,8 @@ spec: containers: - name: telegram-scraper image: gcr.forust.xyz/forust/telegram-scraper:latest - tty: true stdin: true + tty: true ports: - containerPort: 8080 livenessProbe: @@ -59,14 +59,16 @@ spec: - metadata: name: telegram-scraper-data spec: - accessModes: ["ReadWriteOnce"] + accessModes: + - ReadWriteOnce resources: requests: storage: 20Gi - metadata: name: telegram-scraper-session spec: - accessModes: ["ReadWriteOnce"] + accessModes: + - ReadWriteOnce resources: requests: storage: 5Mi diff --git a/pyproject.toml b/pyproject.toml index 55d89ab..12ea529 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "telegram-scraper" -version = "0.1.0" +version = "2.0.0" description = "Telegram-scraper with webui" readme = "README.md" requires-python = ">=3.11" diff --git a/telegram_scraper_with_forwarding.py b/telegram_scraper_with_forwarding.py index fe4cc1b..8c4fbcf 100644 --- a/telegram_scraper_with_forwarding.py +++ b/telegram_scraper_with_forwarding.py @@ -23,11 +23,8 @@ from telethon.errors import FloodWaitError, SessionPasswordNeededError import qrcode from app_state import ( StateStore, - account_data_dir, account_session_path, get_account_store, - load_account, - save_account, ) warnings.filterwarnings( diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..c07ad83 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,604 @@ +""" +Smoke / integration tests for the multi-account Telegram scraper core logic. + +These tests do NOT require Telegram credentials or network access. +They verify: + - Account creation and state isolation + - Account import safety (no auto-start continuous) + - Account export round-trip + - Account deletion (sidecar file cleanup) + - Active account persistence helpers + - Message search SQL construction + - Health payload structure + - Continuous orchestrator worker deduplication + - load_messages() pagination with search filter +""" + +import os +import shutil +import sqlite3 +import sys +import tempfile +import threading +import time +from pathlib import Path +from typing import Any, Dict, List, Optional +from unittest.mock import MagicMock + +# ── Mock heavy dependencies before any webui_server import ───────────── +# We need to mock qrcode and telethon because they aren't installed in CI. +sys.modules["qrcode"] = MagicMock() +sys.modules["qrcode.image"] = MagicMock() +sys.modules["qrcode.image.svg"] = MagicMock() +sys.modules["telethon"] = MagicMock() +sys.modules["telethon.errors"] = MagicMock() +sys.modules["telethon.errors"].SessionPasswordNeededError = type("SessionPasswordNeededError", (Exception,), {}) +sys.modules["telegram_scraper_with_forwarding"] = MagicMock() +sys.modules["telegram_scraper_with_forwarding"]._ensure_session_wal = lambda p: None +sys.modules["telegram_scraper_with_forwarding"].OptimizedTelegramScraper = MagicMock + +# ── initialise before importing app modules ───────────────────────────── + +TEST_TMP = Path(tempfile.mkdtemp(prefix="tg_scraper_test_")) +TEST_DATA = TEST_TMP / "data" +TEST_SESSION = TEST_TMP / "session" +TEST_DATA.mkdir(parents=True, exist_ok=True) +TEST_SESSION.mkdir(parents=True, exist_ok=True) + +os.environ["TELEGRAM_SCRAPER_HOST"] = "127.0.0.1" +os.environ["TELEGRAM_SCRAPER_PORT"] = "0" +os.environ["TELEGRAM_SCRAPER_START_CONTINUOUS"] = "0" + +# Now safe to import app modules +import app_state # noqa: E402 +from app_state import ( # noqa: E402 + ACCOUNT_DEFAULTS, + account_data_dir, + account_exists, + account_session_path, + get_account_store, + get_global_store, + list_accounts, + load_account, +) +from health import health_payload # noqa: E402 + + +def _init_test_env(): + """Reset global caches and re-initialise for a clean test.""" + app_state._GLOBAL_STORE = None + with app_state._account_stores_lock: + app_state._ACCOUNT_STORES.clear() + + +_init_test_env() + + +# ── Helpers ──────────────────────────────────────────────────────────────── + + +def make_account_id() -> str: + return f"test-{int(time.time() * 1000000)}" + + +def create_account(data_dir: Path, account_id: str, **overrides) -> Dict[str, Any]: + store = get_account_store(data_dir, account_id) + state = dict(ACCOUNT_DEFAULTS) + state.update(overrides) + state["label"] = overrides.get("label", account_id) + store.save(state) + + def mutate(g: Dict[str, Any]) -> None: + accounts = g.setdefault("accounts", []) + if account_id not in accounts: + accounts.append(account_id) + + get_global_store(data_dir).update(mutate) + return load_account(data_dir, account_id) + + +def create_channel_db(data_dir: Path, account_id: Optional[str], channel_id: str, messages: List[Dict[str, Any]]) -> None: + if account_id: + db_dir = account_data_dir(data_dir, account_id) / channel_id + else: + db_dir = data_dir / channel_id + db_dir.mkdir(parents=True, exist_ok=True) + db_path = db_dir / f"{channel_id}.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + """CREATE TABLE IF NOT EXISTS messages ( + message_id INTEGER PRIMARY KEY, + date TEXT, sender_id INTEGER, + first_name TEXT, last_name TEXT, username TEXT, + message TEXT, media_type TEXT, media_path TEXT, + reply_to INTEGER, post_author TEXT, + views INTEGER, forwards INTEGER, reactions TEXT + )""" + ) + for m in messages: + conn.execute( + "INSERT OR IGNORE INTO messages (message_id, date, message) VALUES (?, ?, ?)", + (m["message_id"], m.get("date"), m.get("message")), + ) + conn.commit() + conn.close() + + +# ── Fixture setup / teardown ──────────────────────────────────────────────── + + +def setup_function(): + _init_test_env() + for child in TEST_DATA.iterdir(): + if child.is_dir(): + shutil.rmtree(child, ignore_errors=True) + else: + child.unlink(missing_ok=True) + + +def teardown_function(): + _init_test_env() + + +# ═══════════════════════════════════════════════════════════════════════════ +# Tests +# ═══════════════════════════════════════════════════════════════════════════ + + +class TestAccountLifecycle: + """Account creation, isolation, import, export, deletion.""" + + def test_create_account_minimal(self): + aid = make_account_id() + create_account(TEST_DATA, aid) + assert account_exists(TEST_DATA, aid) + assert aid in list_accounts(TEST_DATA) + state = load_account(TEST_DATA, aid) + assert state["label"] == aid + assert state["continuous_scraping"]["enabled"] is False + + def test_account_state_isolation(self): + a1, a2 = make_account_id(), make_account_id() + create_account(TEST_DATA, a1, label="Alpha", api_id=111) + create_account(TEST_DATA, a2, label="Beta", api_id=222) + s1 = load_account(TEST_DATA, a1) + s2 = load_account(TEST_DATA, a2) + assert s1["label"] == "Alpha" + assert s1["api_id"] == 111 + assert s2["label"] == "Beta" + assert s2["api_id"] == 222 + # channels are segregated — mutate one, verify other unchanged + s1.setdefault("channels", {})["-100aaa"] = 1 + get_account_store(TEST_DATA, a1).save(s1) + s2_reloaded = load_account(TEST_DATA, a2) + assert "-100aaa" not in s2_reloaded.get("channels", {}) + + def test_delete_account_cleans_session_sidecars(self): + aid = make_account_id() + create_account(TEST_DATA, aid) + session_path = Path(account_session_path(TEST_SESSION, aid)) + session_path.write_text("session-data") + Path(str(session_path) + "-wal").write_text("wal") + Path(str(session_path) + "-shm").write_text("shm") + Path(str(session_path) + "-journal").write_text("journal") + + acc_dir = account_data_dir(TEST_DATA, aid) + if acc_dir.exists(): + shutil.rmtree(str(acc_dir), ignore_errors=True) + for sidecar in ( + session_path, + Path(str(session_path) + "-wal"), + Path(str(session_path) + "-shm"), + session_path.with_suffix(session_path.suffix + "-journal"), + ): + if sidecar.exists(): + sidecar.unlink() + + assert not session_path.exists() + assert not Path(str(session_path) + "-wal").exists() + assert not Path(str(session_path) + "-shm").exists() + assert not Path(str(session_path) + "-journal").exists() + assert not acc_dir.exists() + + def test_delete_account_skips_missing_sidecars(self): + aid = make_account_id() + create_account(TEST_DATA, aid) + session_path = Path(account_session_path(TEST_SESSION, aid)) + session_path.write_text("session-only") + acc_dir = account_data_dir(TEST_DATA, aid) + if acc_dir.exists(): + shutil.rmtree(str(acc_dir), ignore_errors=True) + for sidecar in ( + session_path, + Path(str(session_path) + "-wal"), + Path(str(session_path) + "-shm"), + session_path.with_suffix(session_path.suffix + "-journal"), + ): + if sidecar.exists(): + sidecar.unlink() + assert not session_path.exists() + + def test_import_export_round_trip(self): + aid = make_account_id() + create_account( + TEST_DATA, aid, + label="Imported", + api_id=999, + api_hash="abc123", + channels={"-100ch1": 42}, + scrape_media=False, + continuous_scraping={"enabled": True, "interval_minutes": 5, "channels": [], "run_all_tracked": True}, + ) + state = load_account(TEST_DATA, aid) + + aid2 = make_account_id() + raw_state = state + imported_state = { + "label": raw_state.get("label", aid2), + "api_id": raw_state.get("api_id"), + "api_hash": raw_state.get("api_hash"), + "channels": raw_state.get("channels", {}), + "channel_names": raw_state.get("channel_names", {}), + "scrape_media": bool(raw_state.get("scrape_media", True)), + "forwarding_rules": raw_state.get("forwarding_rules", []), + "continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else { + "enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True, + }, + } + get_account_store(TEST_DATA, aid2).save(imported_state) + + def mutate_global(g): + g.setdefault("accounts", []).append(aid2) + get_global_store(TEST_DATA).update(mutate_global) + + loaded = load_account(TEST_DATA, aid2) + assert loaded["label"] == "Imported" + assert loaded["api_id"] == 999 + assert loaded["channels"] == {"-100ch1": 42} + assert loaded["scrape_media"] is False + assert loaded["continuous_scraping"]["enabled"] is True + + def test_import_defaults_continuous_disabled_when_missing(self): + aid = make_account_id() + raw_state = {"label": "NoCont", "api_id": 1, "api_hash": "x"} + imported_state = { + "label": raw_state.get("label", aid), + "api_id": raw_state.get("api_id"), + "api_hash": raw_state.get("api_hash"), + "channels": {}, + "channel_names": {}, + "scrape_media": True, + "forwarding_rules": [], + "continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else { + "enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True, + }, + } + get_account_store(TEST_DATA, aid).save(imported_state) + + def mutate_global(g): + g.setdefault("accounts", []).append(aid) + get_global_store(TEST_DATA).update(mutate_global) + + loaded = load_account(TEST_DATA, aid) + assert loaded["continuous_scraping"]["enabled"] is False + assert loaded["continuous_scraping"]["run_all_tracked"] is True + + +class TestContinuousOrchestrator: + """Continuous scraping manager safety using mocked webui_server.""" + + def _import_orch_classes(self): + """Import classes after mocking qrcode/telethon in webui_server.""" + import webui_server as ws_module + return ws_module.PerAccountContinuousScrapeManager, ws_module.ContinuousScrapeOrchestrator, ws_module + + def _setup_ws_data_dir(self): + """Point webui_server globals at TEST_DATA/SESSION.""" + import webui_server as ws_module + self._ws_orig_data = ws_module.DATA_DIR + self._ws_orig_session = ws_module.SESSION_DIR + ws_module.DATA_DIR = TEST_DATA + ws_module.SESSION_DIR = TEST_SESSION + ws_module.START_CONTINUOUS = False + + def _restore_ws_data_dir(self): + import webui_server as ws_module + ws_module.DATA_DIR = self._ws_orig_data + ws_module.SESSION_DIR = self._ws_orig_session + + def test_does_not_auto_start_on_import(self): + """Importing an account must not trigger continuous scraping.""" + PerAccountContinuousScrapeManager, ContinuousScrapeOrchestrator, ws = self._import_orch_classes() + self._setup_ws_data_dir() + try: + orch = ContinuousScrapeOrchestrator() + aid = make_account_id() + create_account(TEST_DATA, aid, continuous_scraping={ + "enabled": True, "interval_minutes": 1, "channels": [], "run_all_tracked": True, + }) + orch.add_account(aid, auto_start=False) + snap = orch.snapshot_for(aid) + assert snap["status"]["running"] is False + finally: + self._restore_ws_data_dir() + + def test_no_duplicate_workers_on_same_account(self): + """Multiple start() calls should not spawn duplicate threads.""" + PerAccountContinuousScrapeManager, _, ws = self._import_orch_classes() + self._setup_ws_data_dir() + try: + aid = make_account_id() + # Must set enabled=True or refresh_config() will stop the thread + create_account(TEST_DATA, aid, continuous_scraping={ + "enabled": True, "interval_minutes": 60, "channels": [], "run_all_tracked": True, + }) + mgr = PerAccountContinuousScrapeManager(aid) + # Override refresh_config to prevent auth check from interfering + original_refresh = mgr.refresh_config + mgr.refresh_config = lambda: None + mgr.start() + t1 = mgr.thread + mgr.start() # second call — should be no-op + t2 = mgr.thread + assert t1 is t2, "start() spawned a second thread" + # Also verify only one thread is alive for this manager + alive_count = sum( + 1 for t in threading.enumerate() + if t is t1 or t is t2 + ) + assert alive_count <= 1, "duplicate threads detected" + mgr.stop() + mgr.refresh_config = original_refresh + finally: + self._restore_ws_data_dir() + + def test_disabled_account_not_started(self): + """start_all() must not start accounts with enabled=False.""" + _, ContinuousScrapeOrchestrator, ws = self._import_orch_classes() + self._setup_ws_data_dir() + try: + orch = ContinuousScrapeOrchestrator() + aid = make_account_id() + create_account(TEST_DATA, aid, continuous_scraping={ + "enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True, + }) + orch.start_all() + snap = orch.snapshot_for(aid) + assert snap["status"]["running"] is False + finally: + self._restore_ws_data_dir() + + +class TestMessageSearch: + """load_messages() SQL correctness with pagination and search.""" + + def _import_load_messages(self): + import webui_server as ws_module + # Point at TEST_DATA + self._ws_orig = ws_module.DATA_DIR + ws_module.DATA_DIR = TEST_DATA + return ws_module.load_messages + + def _restore(self): + import webui_server as ws_module + ws_module.DATA_DIR = self._ws_orig + + def test_search_filters_correctly(self): + channel_id = "-100searchtest" + messages = [ + {"message_id": 1, "date": "2024-01-01", "message": "hello world"}, + {"message_id": 2, "date": "2024-01-02", "message": "foo bar baz"}, + {"message_id": 3, "date": "2024-01-03", "message": "hello again"}, + ] + create_channel_db(TEST_DATA, "default", channel_id, messages) + load_messages = self._import_load_messages() + try: + result = load_messages("default", channel_id, limit=100, search="hello") + finally: + self._restore() + texts = [m["text"] for m in result] + assert "hello world" in texts + assert "hello again" in texts + assert "foo bar baz" not in texts + + def test_before_id_pagination_works(self): + channel_id = "-100pagination" + messages = [ + {"message_id": i, "date": f"2024-01-{i:02d}", "message": f"msg-{i}"} + for i in range(1, 21) + ] + create_channel_db(TEST_DATA, "default", channel_id, messages) + load_messages = self._import_load_messages() + try: + result = load_messages("default", channel_id, limit=5, before_message_id=15) + finally: + self._restore() + assert len(result) == 5 + ids = [m["message_id"] for m in result] + assert all(i < 15 for i in ids) + # load_messages returns rows in ascending message_id order (reversed from DESC query) + assert ids == [10, 11, 12, 13, 14] + + def test_search_with_pagination(self): + channel_id = "-100searchpages" + messages = [ + {"message_id": i, "date": f"2024-01-{i:02d}", "message": f"hello-{i}" if i % 2 else f"other-{i}"} + for i in range(1, 21) + ] + create_channel_db(TEST_DATA, "default", channel_id, messages) + load_messages = self._import_load_messages() + try: + result = load_messages("default", channel_id, limit=3, before_message_id=15, search="hello") + finally: + self._restore() + # IDs matching "hello-" below 15: 13, 11, 9, 7, 5, 3, 1 → LIMIT 3 → [9, 11, 13] (ascending) + ids = [m["message_id"] for m in result] + assert ids == [9, 11, 13], f"got {ids}" + assert all("hello" in m["text"] for m in result) + + def test_empty_result_no_error(self): + channel_id = "-100emptysearch" + messages = [{"message_id": 1, "date": "2024-01-01", "message": "only one"}] + create_channel_db(TEST_DATA, "default", channel_id, messages) + load_messages = self._import_load_messages() + try: + result = load_messages("default", channel_id, limit=100, search="nonexistent") + finally: + self._restore() + assert result == [] + + +class TestHealthPayload: + """Health endpoint structure.""" + + def test_health_payload_structure(self): + aid = make_account_id() + create_account(TEST_DATA, aid) + payload = health_payload( + TEST_DATA, + TEST_SESSION, + get_global_store(TEST_DATA), + {}, + 0, + [aid], + ) + assert "ok" in payload + assert "status" in payload + assert "checks" in payload + checks = payload["checks"] + assert "data_dir" in checks + assert "session_dir" in checks + assert "accounts" in checks + assert "items" in checks["accounts"] + assert aid in checks["accounts"]["items"] + + def test_health_accounts_wrapper(self): + a1, a2 = make_account_id(), make_account_id() + create_account(TEST_DATA, a1) + create_account(TEST_DATA, a2) + payload = health_payload( + TEST_DATA, TEST_SESSION, + get_global_store(TEST_DATA), + {}, 0, [a1, a2], + ) + acc_checks = payload["checks"]["accounts"] + assert acc_checks["ok"] is True + assert len(acc_checks["items"]) == 2 + + +class TestPersistence: + """UI persistence helpers.""" + + def test_active_account_localstorage(self): + """Verifies the save/load logic pattern used in app.js.""" + accounts = [ + {"id": "work", "label": "Work"}, + {"id": "home", "label": "Home"}, + ] + saved_id = "work" + loaded = None + if saved_id and any(a["id"] == saved_id for a in accounts): + loaded = saved_id + if not loaded: + loaded = next((a["id"] for a in accounts), None) + assert loaded == "work" + + saved_id = "longgone" + loaded = None + if saved_id and any(a["id"] == saved_id for a in accounts): + loaded = saved_id + if not loaded and accounts: + loaded = accounts[0]["id"] + assert loaded == "work" + + def test_viewer_fallback_on_missing_account(self): + """Viewer's loadViewerAccount fallback logic.""" + accounts = [ + {"id": "alpha", "label": "Alpha"}, + {"id": "beta", "label": "Beta"}, + ] + requested = "nonexistent" + requested_exists = requested and any(a["id"] == requested for a in accounts) + account_id = requested if requested_exists else (accounts[0]["id"] if accounts else None) + assert account_id == "alpha" + + account_id2 = requested if (requested and any(a["id"] == requested for a in [])) else None + assert account_id2 is None + + +class TestJobDeduplicationSchema: + """JobRunner create_job dedup logic.""" + + def _import_job_runner(self): + import webui_server as ws_module + return ws_module.JobRunner + + def test_create_job_dedup_by_account(self): + JobRunner = self._import_job_runner() + runner = JobRunner() + j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"}) + j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"}) + assert j1.job_id == j2.job_id, "duplicate job was created instead of being reused" + assert "Reused existing active job" in j2.logs + + def test_create_job_allows_different_accounts(self): + JobRunner = self._import_job_runner() + runner = JobRunner() + j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"}) + time.sleep(0.002) # ensure different timestamp -> different job_id + j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc2"}) + assert j1.job_id != j2.job_id + + def test_create_job_allows_after_previous_completes(self): + JobRunner = self._import_job_runner() + runner = JobRunner() + j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"}) + j1.status = "completed" + time.sleep(0.002) # ensure different timestamp -> different job_id + j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"}) + assert j1.job_id != j2.job_id + + +class TestAccountHealthSummary: + """account_health_summary structure.""" + + def _import_health_summary(self): + import webui_server as ws_module + self._ws_orig_data = ws_module.DATA_DIR + self._ws_orig_session = ws_module.SESSION_DIR + ws_module.DATA_DIR = TEST_DATA + ws_module.SESSION_DIR = TEST_SESSION + return ws_module.account_health_summary, ws_module.JobRunner, ws_module + + def test_health_summary_structure(self): + account_health_summary, JobRunner, ws = self._import_health_summary() + aid = make_account_id() + create_account(TEST_DATA, aid, api_id=1, api_hash="x") + try: + runner = JobRunner() + health = account_health_summary(aid, runner) + finally: + ws.DATA_DIR = self._ws_orig_data + ws.SESSION_DIR = self._ws_orig_session + + assert health["account_id"] == aid + assert "label" in health + assert "data_dir_exists" in health + assert "session_ready" in health + assert "api_credentials" in health + assert health["api_credentials"] is True + assert "channel_count" in health + assert "message_count" in health + assert "media_count" in health + + +# ── Cleanup all temp data ────────────────────────────────────────────────── + + +def cleanup_test_data(): + if TEST_TMP.exists(): + shutil.rmtree(str(TEST_TMP), ignore_errors=True) + + +import atexit # noqa: E402 +atexit.register(cleanup_test_data) # noqa: E402 diff --git a/webui/app.js b/webui/app.js index 1fabdfb..ce30470 100644 --- a/webui/app.js +++ b/webui/app.js @@ -10,16 +10,19 @@ const state = { authStates: {}, }; +const ACTIVE_ACCOUNT_STORAGE_KEY = 'telegramScraper.activeAccount'; +const jobStreams = new Map(); + /* ── API helper ─────────────────────────────────────── */ async function api(path, options = {}) { const response = await fetch(path, { - headers: { "Content-Type": "application/json" }, + headers: { 'Content-Type': 'application/json' }, ...options, }); const data = await response.json(); if (!response.ok) { - throw new Error(data.error || "Request failed"); + throw new Error(data.error || 'Request failed'); } return data; } @@ -27,16 +30,16 @@ async function api(path, options = {}) { /* ── Format helpers ─────────────────────────────────── */ function formatDate(value) { - if (!value) return "-"; - return value.replace("T", " ").replace("+00:00", " UTC"); + if (!value) return '-'; + return value.replace('T', ' ').replace('+00:00', ' UTC'); } function relativeTime(value) { - if (!value) return "-"; + if (!value) return '-'; const date = new Date(value); - if (Number.isNaN(date.getTime())) return "-"; + if (Number.isNaN(date.getTime())) return '-'; const seconds = Math.max(0, Math.floor((Date.now() - date.getTime()) / 1000)); - if (seconds < 10) return "now"; + if (seconds < 10) return 'now'; if (seconds < 60) return `${seconds}s ago`; const minutes = Math.floor(seconds / 60); if (minutes < 60) return `${minutes}m ago`; @@ -46,16 +49,16 @@ function relativeTime(value) { } function displayTime(value) { - if (!value) return "-"; + if (!value) return '-'; const date = new Date(value); if (Number.isNaN(date.getTime())) return formatDate(value); return date.toLocaleString(undefined, { - year: "numeric", - month: "2-digit", - day: "2-digit", - hour: "2-digit", - minute: "2-digit", - second: "2-digit", + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', }); } @@ -68,44 +71,103 @@ function confirmAction(message) { return window.confirm(message); } +function showToast(message, type = 'info') { + let root = document.getElementById('toast-root'); + if (!root) { + root = document.createElement('div'); + root.id = 'toast-root'; + root.className = 'toast-root'; + document.body.appendChild(root); + } + const toast = document.createElement('div'); + toast.className = `toast toast-${type}`; + toast.textContent = message; + root.appendChild(toast); + window.setTimeout(() => { + toast.classList.add('toast-hide'); + window.setTimeout(() => toast.remove(), 250); + }, 3600); +} + +function downloadJson(filename, payload) { + const blob = new Blob([JSON.stringify(payload, null, 2) + '\n'], { + type: 'application/json', + }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); +} + function isAccountAuthorized(auth) { const status = auth?.auth_status || auth || {}; return ( - auth?.phase === "authorized" || - auth?.status === "authorized" || - status.status === "ready" || - status.status === "authorized" + auth?.phase === 'authorized' || + auth?.status === 'authorized' || + status.status === 'ready' || + status.status === 'authorized' ); } +function saveActiveAccount(accountId) { + if (accountId) { + localStorage.setItem(ACTIVE_ACCOUNT_STORAGE_KEY, accountId); + } +} + +function loadSavedActiveAccount(accounts) { + let savedId = localStorage.getItem(ACTIVE_ACCOUNT_STORAGE_KEY); + if (!savedId) { + savedId = localStorage.getItem('activeAccount'); + if (savedId) { + localStorage.setItem(ACTIVE_ACCOUNT_STORAGE_KEY, savedId); + localStorage.removeItem('activeAccount'); + } + } + if (savedId && accounts.some((account) => account.id === savedId)) { + return savedId; + } + if (savedId) { + localStorage.removeItem(ACTIVE_ACCOUNT_STORAGE_KEY); + } + if (state.activeAccount && accounts.some((account) => account.id === state.activeAccount)) { + return state.activeAccount; + } + return accounts[0]?.id || null; +} + /* ── Account tab management ─────────────────────────── */ function renderAccountTabs(accounts) { - const container = document.getElementById("account-tabs"); - const template = document.getElementById("account-tab-template"); - container.innerHTML = ""; + const container = document.getElementById('account-tabs'); + const template = document.getElementById('account-tab-template'); + container.innerHTML = ''; accounts.forEach((acc) => { const node = template.content.firstElementChild.cloneNode(true); node.dataset.accountId = acc.id; - node.querySelector(".account-tab-name").textContent = acc.label || acc.id; - const statusEl = node.querySelector(".account-tab-status"); + node.querySelector('.account-tab-name').textContent = acc.label || acc.id; + const statusEl = node.querySelector('.account-tab-status'); if (acc.auth) { const authOk = isAccountAuthorized(acc.auth); - statusEl.textContent = authOk ? "●" : "○"; - statusEl.className = "account-tab-status " + (authOk ? "ok" : ""); + statusEl.textContent = authOk ? '●' : '○'; + statusEl.className = 'account-tab-status ' + (authOk ? 'ok' : ''); } if (acc.id === state.activeAccount) { - node.classList.add("active"); + node.classList.add('active'); } - node.addEventListener("click", () => switchAccount(acc.id)); + node.addEventListener('click', () => switchAccount(acc.id)); container.appendChild(node); }); } function renderAccountPanel(accountId) { - const container = document.getElementById("account-panels"); - const template = document.getElementById("account-panel-template"); + const container = document.getElementById('account-panels'); + const template = document.getElementById('account-panel-template'); // Don't duplicate if (document.getElementById(`panel-${accountId}`)) return; @@ -115,14 +177,14 @@ function renderAccountPanel(accountId) { node.dataset.accountId = accountId; // Add/remove channel form - const addForm = node.querySelector(".add-channel-form"); - addForm.addEventListener("submit", async (event) => { + const addForm = node.querySelector('.add-channel-form'); + addForm.addEventListener('submit', async (event) => { event.preventDefault(); - const channelId = addForm.querySelector(".add-channel-id").value.trim(); - const name = addForm.querySelector(".add-channel-name").value.trim(); + const channelId = addForm.querySelector('.add-channel-id').value.trim(); + const name = addForm.querySelector('.add-channel-name').value.trim(); if (!channelId) return; await api(`/api/accounts/${accountId}/channels/add`, { - method: "POST", + method: 'POST', body: JSON.stringify({ channel_id: channelId, name }), }); addForm.reset(); @@ -130,63 +192,76 @@ function renderAccountPanel(accountId) { }); // Continuous form - const contForm = node.querySelector(".continuous-form"); - contForm.addEventListener("submit", async (event) => { + const contForm = node.querySelector('.continuous-form'); + contForm.addEventListener('submit', async (event) => { event.preventDefault(); - const enabled = contForm.querySelector(".continuous-enabled").checked; - const intervalMinutes = contForm.querySelector(".continuous-interval").value; - const runAllTracked = contForm.querySelector(".continuous-all").checked; - const channels = Array.from(contForm.querySelectorAll(".continuous-channel-checkbox:checked")).map( - (item) => item.value + const enabled = contForm.querySelector('.continuous-enabled').checked; + const intervalMinutes = contForm.querySelector('.continuous-interval').value; + const runAllTracked = contForm.querySelector('.continuous-all').checked; + const channels = Array.from(contForm.querySelectorAll('.continuous-channel-checkbox:checked')).map( + (item) => item.value, ); if (!runAllTracked && channels.length === 0 && enabled) { - if (!confirmAction("Continuous scraping enabled with no selected channels. Save anyway?")) return; + if (!confirmAction('Continuous scraping enabled with no selected channels. Save anyway?')) return; } await api(`/api/accounts/${accountId}/continuous`, { - method: "POST", + method: 'POST', body: JSON.stringify({ enabled, interval_minutes: intervalMinutes, run_all_tracked: runAllTracked, channels }), }); await refreshAccount(accountId); }); // "All tracked" checkbox toggles individual checkboxes - const contAll = contForm.querySelector(".continuous-all"); - contAll.addEventListener("change", () => { - const checkboxes = contForm.querySelectorAll(".continuous-channel-checkbox"); - checkboxes.forEach((cb) => { cb.disabled = contAll.checked; }); + const contAll = contForm.querySelector('.continuous-all'); + contAll.addEventListener('change', () => { + const checkboxes = contForm.querySelectorAll('.continuous-channel-checkbox'); + checkboxes.forEach((cb) => { + cb.disabled = contAll.checked; + }); }); // Content tabs within account panel - const contentTabs = node.querySelectorAll(".content-tab"); + const contentTabs = node.querySelectorAll('.content-tab'); contentTabs.forEach((tab) => { - tab.addEventListener("click", () => { + tab.addEventListener('click', () => { const targetId = tab.dataset.tabTarget; - contentTabs.forEach((t) => t.classList.toggle("active", t === tab)); - node.querySelectorAll(".tab-panel").forEach((p) => { - p.classList.toggle("active", p.dataset.panel === targetId); + contentTabs.forEach((t) => t.classList.toggle('active', t === tab)); + node.querySelectorAll('.tab-panel').forEach((p) => { + p.classList.toggle('active', p.dataset.panel === targetId); }); }); }); // Refresh jobs - node.querySelector(".refresh-jobs-btn").addEventListener("click", () => refreshAccount(accountId)); + node.querySelector('.refresh-jobs-btn').addEventListener('click', () => refreshAccount(accountId)); container.appendChild(node); } +function closeAllJobStreams() { + for (const [jobId, handle] of jobStreams) { + if (handle instanceof EventSource || typeof handle?.close === 'function') { + handle.close(); + } else if (typeof handle === 'number') { + clearInterval(handle); + } + } + jobStreams.clear(); +} + function switchAccount(accountId) { - if (state.activeAccount === accountId) return; + closeAllJobStreams(); state.activeAccount = accountId; - localStorage.setItem("activeAccount", accountId); + saveActiveAccount(accountId); // Update tabs - document.querySelectorAll(".account-tab").forEach((tab) => { - tab.classList.toggle("active", tab.dataset.accountId === accountId); + document.querySelectorAll('.account-tab').forEach((tab) => { + tab.classList.toggle('active', tab.dataset.accountId === accountId); }); // Show/hide panels - document.querySelectorAll(".account-panel").forEach((panel) => { - panel.classList.toggle("hidden", panel.dataset.accountId !== accountId); + document.querySelectorAll('.account-panel').forEach((panel) => { + panel.classList.toggle('hidden', panel.dataset.accountId !== accountId); }); // Ensure panel exists @@ -206,17 +281,17 @@ function switchAccount(accountId) { function updateSidebarAccount() { const acc = state.accounts.find((a) => a.id === state.activeAccount); - const nameEl = document.getElementById("active-account-name"); - const statusEl = document.getElementById("active-account-status"); + const nameEl = document.getElementById('active-account-name'); + const statusEl = document.getElementById('active-account-status'); if (acc) { nameEl.textContent = acc.label || acc.id; const authOk = isAccountAuthorized(acc.auth); - statusEl.textContent = authOk ? "Authorized session" : "Needs login"; - statusEl.style.color = authOk ? "var(--ok)" : "var(--danger)"; + statusEl.textContent = authOk ? 'Authorized session' : 'Needs login'; + statusEl.style.color = authOk ? 'var(--ok)' : 'var(--danger)'; } else { - nameEl.textContent = "None"; - statusEl.textContent = "Add an account in Settings"; - statusEl.style.color = "var(--dim)"; + nameEl.textContent = 'None'; + statusEl.textContent = 'Add an account in Settings'; + statusEl.style.color = 'var(--dim)'; } } @@ -225,50 +300,57 @@ function updateSidebarAccount() { function renderChannels(accountId, channels) { const panel = document.getElementById(`panel-${accountId}`); if (!panel) return; - const tbody = panel.querySelector(".channels-table"); - const template = document.getElementById("channel-row-template"); - tbody.innerHTML = ""; + const tbody = panel.querySelector('.channels-table'); + const template = document.getElementById('channel-row-template'); + tbody.innerHTML = ''; + + if (!channels.length) { + const row = document.createElement('tr'); + row.innerHTML = + '
No tracked channels yet. Add an ID or @username to start scraping this account.
'; + tbody.appendChild(row); + } channels.forEach((channel) => { const node = template.content.firstElementChild.cloneNode(true); - node.querySelector(".channel-name").textContent = channel.name; - node.querySelector(".channel-id").textContent = channel.channel_id; - node.querySelector(".message-count").textContent = String(channel.message_count); - node.querySelector(".media-count").textContent = String(channel.media_count); - node.querySelector(".last-date").textContent = channel.last_date || "-"; + node.querySelector('.channel-name').textContent = channel.name; + node.querySelector('.channel-id').textContent = channel.channel_id; + node.querySelector('.message-count').textContent = String(channel.message_count); + node.querySelector('.media-count').textContent = String(channel.media_count); + node.querySelector('.last-date').textContent = channel.last_date || '-'; - node.querySelector(".scrape-btn").addEventListener("click", async () => { + node.querySelector('.scrape-btn').addEventListener('click', async () => { await api(`/api/accounts/${accountId}/jobs/scrape`, { - method: "POST", + method: 'POST', body: JSON.stringify({ channel_id: channel.channel_id }), }); await refreshAccount(accountId); }); - node.querySelector(".export-btn").addEventListener("click", async () => { + node.querySelector('.export-btn').addEventListener('click', async () => { await api(`/api/accounts/${accountId}/jobs/export-channel`, { - method: "POST", + method: 'POST', body: JSON.stringify({ channel_id: channel.channel_id }), }); await refreshAccount(accountId); }); - node.querySelector(".export-view-btn").addEventListener("click", () => { + node.querySelector('.export-view-btn').addEventListener('click', () => { window.location.href = `/viewer?channel=${encodeURIComponent(channel.channel_id)}&account=${encodeURIComponent(accountId)}`; }); - node.querySelector(".media-btn").addEventListener("click", async () => { + node.querySelector('.media-btn').addEventListener('click', async () => { await api(`/api/accounts/${accountId}/jobs/rescrape-media`, { - method: "POST", + method: 'POST', body: JSON.stringify({ channel_id: channel.channel_id }), }); await refreshAccount(accountId); }); - node.querySelector(".remove-btn").addEventListener("click", async () => { + node.querySelector('.remove-btn').addEventListener('click', async () => { if (!confirmAction(`Remove ${channel.name} from tracked channels?`)) return; await api(`/api/accounts/${accountId}/channels/remove`, { - method: "POST", + method: 'POST', body: JSON.stringify({ channel_id: channel.channel_id }), }); await refreshAccount(accountId); @@ -278,64 +360,159 @@ function renderChannels(accountId, channels) { }); // Update channel count stat - panel.querySelector(".channel-count").textContent = String(channels.length); + panel.querySelector('.channel-count').textContent = String(channels.length); } function renderJobs(accountId, jobs) { const panel = document.getElementById(`panel-${accountId}`); if (!panel) return; - const root = panel.querySelector(".jobs-list"); - const template = document.getElementById("job-template"); - root.innerHTML = ""; + const root = panel.querySelector('.jobs-list'); + const template = document.getElementById('job-template'); + root.innerHTML = ''; if (!jobs.length) { - root.innerHTML = '
No jobs yet.
'; + root.innerHTML = '
No jobs yet. Start a scrape or export to see progress here.
'; return; } jobs.forEach((job) => { const node = template.content.firstElementChild.cloneNode(true); - node.querySelector(".job-title").textContent = job.title; - node.querySelector(".job-status").textContent = job.status; - node.querySelector(".job-time").textContent = [displayTime(job.started_at), displayTime(job.finished_at)] - .filter((item) => item !== "-") - .join(" -> "); - node.querySelector(".job-logs").textContent = job.logs || job.error || "No logs."; + node.dataset.jobId = job.job_id; + node.querySelector('.job-title').textContent = job.title; + node.querySelector('.job-status').textContent = job.status; + node.querySelector('.job-time').textContent = [displayTime(job.started_at), displayTime(job.finished_at)] + .filter((item) => item !== '-') + .join(' -> '); + node.querySelector('.job-logs').textContent = job.logs || job.error || 'No logs.'; root.appendChild(node); + subscribeJobStream(accountId, job.job_id, job.status); }); } +function updateRenderedJob(accountId, job) { + const panel = document.getElementById(`panel-${accountId}`); + const node = panel?.querySelector(`[data-job-id="${job.job_id}"]`); + if (!node) return; + node.querySelector('.job-status').textContent = job.status; + node.querySelector('.job-time').textContent = [displayTime(job.started_at), displayTime(job.finished_at)] + .filter((item) => item !== '-') + .join(' -> '); + node.querySelector('.job-logs').textContent = job.logs || job.error || 'No logs.'; +} + +function pollJobFallback(accountId, jobId) { + let pollAttempts = 0; + const maxPollAttempts = 60; + const pollInterval = 2000; + const pollTimer = window.setInterval(async () => { + pollAttempts++; + try { + const job = await api(`/api/jobs/${encodeURIComponent(jobId)}`); + updateRenderedJob(accountId, job); + if (['completed', 'failed'].includes(job.status)) { + clearInterval(pollTimer); + jobStreams.delete(jobId); + refreshAccount(accountId); + return; + } + } catch { + // ignore poll errors + } + if (pollAttempts >= maxPollAttempts) { + clearInterval(pollTimer); + jobStreams.delete(jobId); + } + }, pollInterval); + jobStreams.set(jobId, pollTimer); +} + +function subscribeJobStream(accountId, jobId, status) { + if (!['queued', 'running'].includes(status) || jobStreams.has(jobId)) { + return; + } + + // If EventSource not available, fall back to polling + if (!window.EventSource) { + pollJobFallback(accountId, jobId); + return; + } + + let retryCount = 0; + const maxRetries = 5; + let stream = null; + + function connect() { + if (stream) stream.close(); + if (jobStreams.get(jobId) === 'dead') return; + + const newStream = new EventSource(`/api/jobs/${encodeURIComponent(jobId)}/events`); + stream = newStream; + jobStreams.set(jobId, newStream); + + newStream.onmessage = (event) => { + retryCount = 0; // reset backoff on successful message + const job = JSON.parse(event.data); + updateRenderedJob(accountId, job); + if (['completed', 'failed'].includes(job.status)) { + newStream.close(); + jobStreams.delete(jobId); + refreshAccount(accountId); + } + }; + + newStream.onerror = () => { + newStream.close(); + if (jobStreams.get(jobId) === 'dead') return; + + retryCount++; + if (retryCount >= maxRetries) { + // Mark as dead and switch to polling + jobStreams.delete(jobId); + showToast('Live job updates degraded — switching to polling.', 'warn'); + pollJobFallback(accountId, jobId); + return; + } + + // Exponential backoff: 1s, 2s, 4s, 8s, 16s + const delay = Math.min(1000 * Math.pow(2, retryCount - 1), 16000); + setTimeout(connect, delay); + }; + } + + connect(); +} + function renderContinuous(accountId, data) { const panel = document.getElementById(`panel-${accountId}`); if (!panel) return; const config = data.config || {}; const status = data.status || {}; - const contForm = panel.querySelector(".continuous-form"); - contForm.querySelector(".continuous-enabled").checked = Boolean(config.enabled); - contForm.querySelector(".continuous-interval").value = config.interval_minutes || 1; - contForm.querySelector(".continuous-all").checked = Boolean(config.run_all_tracked); + const contForm = panel.querySelector('.continuous-form'); + contForm.querySelector('.continuous-enabled').checked = Boolean(config.enabled); + contForm.querySelector('.continuous-interval').value = config.interval_minutes || 1; + contForm.querySelector('.continuous-all').checked = Boolean(config.run_all_tracked); // Channel picker - const chList = panel.querySelector(".continuous-channel-list"); + const chList = panel.querySelector('.continuous-channel-list'); const channels = state.channels[accountId] || []; const runAll = Boolean(config.run_all_tracked); const selected = new Set(config.channels || []); - chList.innerHTML = ""; + chList.innerHTML = ''; channels.forEach((ch) => { - const label = document.createElement("label"); - label.className = "checkbox-row"; - label.classList.toggle("disabled", runAll); + const label = document.createElement('label'); + label.className = 'checkbox-row'; + label.classList.toggle('disabled', runAll); - const checkbox = document.createElement("input"); - checkbox.type = "checkbox"; - checkbox.className = "continuous-channel-checkbox"; + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + checkbox.className = 'continuous-channel-checkbox'; checkbox.value = ch.channel_id; checkbox.checked = runAll || selected.has(ch.channel_id); checkbox.disabled = runAll; - const text = document.createElement("span"); + const text = document.createElement('span'); text.textContent = `${ch.name} (${ch.channel_id})`; label.append(checkbox, text); @@ -343,85 +520,112 @@ function renderContinuous(accountId, data) { }); // Meta - panel.querySelector(".continuous-status").textContent = status.running ? "running" : "stopped"; - panel.querySelector(".continuous-last-iteration").textContent = status.last_iteration_at + panel.querySelector('.continuous-status').textContent = status.running ? 'running' : 'stopped'; + panel.querySelector('.continuous-last-iteration').textContent = status.last_iteration_at ? `${relativeTime(status.last_iteration_at)} (${displayTime(status.last_iteration_at)})` - : "-"; - panel.querySelector(".continuous-last-error").textContent = status.last_error || "-"; + : '-'; + panel.querySelector('.continuous-next-run').textContent = nextContinuousRunLabel(config, status); + panel.querySelector('.continuous-last-error').textContent = status.last_error || '-'; // Logs - const logsRoot = panel.querySelector(".continuous-logs"); + const logsRoot = panel.querySelector('.continuous-logs'); const entries = (status.log_entries || status.logs || []).map(normalizeLogEntry); - logsRoot.innerHTML = ""; + logsRoot.innerHTML = ''; if (!entries.length) { - logsRoot.innerHTML = '
-INFONo logs yet.
'; + logsRoot.innerHTML = + '
-INFONo logs yet.
'; return; } - entries.slice().reverse().forEach((entry) => { - const row = document.createElement("div"); - row.className = `log-line ${entry.level}`; - const time = entry.timestamp ? displayTime(entry.timestamp).split(", ").pop() : entry.legacyTime || "-"; - const absolute = entry.timestamp ? displayTime(entry.timestamp) : entry.legacyTime || "-"; - row.title = absolute; + entries + .slice() + .reverse() + .forEach((entry) => { + const row = document.createElement('div'); + row.className = `log-line ${entry.level}`; + const time = entry.timestamp ? displayTime(entry.timestamp).split(', ').pop() : entry.legacyTime || '-'; + const absolute = entry.timestamp ? displayTime(entry.timestamp) : entry.legacyTime || '-'; + row.title = absolute; - const timeNode = document.createElement("span"); - timeNode.className = "log-time"; - timeNode.textContent = time; + const timeNode = document.createElement('span'); + timeNode.className = 'log-time'; + timeNode.textContent = time; - const ageNode = document.createElement("span"); - ageNode.className = "log-age"; - ageNode.textContent = entry.timestamp ? relativeTime(entry.timestamp) : "-"; + const ageNode = document.createElement('span'); + ageNode.className = 'log-age'; + ageNode.textContent = entry.timestamp ? relativeTime(entry.timestamp) : '-'; - const levelNode = document.createElement("span"); - levelNode.className = "log-level"; - levelNode.textContent = entry.level.toUpperCase(); + const levelNode = document.createElement('span'); + levelNode.className = 'log-level'; + levelNode.textContent = entry.level.toUpperCase(); - const messageNode = document.createElement("span"); - messageNode.className = "log-message"; - messageNode.textContent = entry.message; + const messageNode = document.createElement('span'); + messageNode.className = 'log-message'; + messageNode.textContent = entry.message; - row.append(timeNode, ageNode, levelNode, messageNode); - logsRoot.appendChild(row); - }); + row.append(timeNode, ageNode, levelNode, messageNode); + logsRoot.appendChild(row); + }); } function classifyLog(message, level) { if (level) return level; const normalized = message.toLowerCase(); - if (normalized.includes("failed") || normalized.includes("error") || normalized.includes("traceback")) return "error"; - if (normalized.includes("sleep") || normalized.includes("no channels")) return "warn"; - if (normalized.includes("finished") || normalized.includes("completed")) return "success"; - if (normalized.includes("starting") || normalized.includes("started")) return "info"; - return "debug"; + if (normalized.includes('failed') || normalized.includes('error') || normalized.includes('traceback')) return 'error'; + if (normalized.includes('sleep') || normalized.includes('no channels')) return 'warn'; + if (normalized.includes('finished') || normalized.includes('completed')) return 'success'; + if (normalized.includes('starting') || normalized.includes('started')) return 'info'; + return 'debug'; } function normalizeLogEntry(entry) { - if (typeof entry === "object" && entry !== null) { + if (typeof entry === 'object' && entry !== null) { return { timestamp: entry.timestamp || null, - level: classifyLog(entry.message || "", entry.level), - message: entry.message || "", + level: classifyLog(entry.message || '', entry.level), + message: entry.message || '', }; } - const line = String(entry || ""); + const line = String(entry || ''); const match = line.match(/^\[(\d{2}:\d{2}:\d{2})\]\s?(.*)$/); return { timestamp: null, - legacyTime: match?.[1] || "", + legacyTime: match?.[1] || '', level: classifyLog(match?.[2] || line), message: match?.[2] || line, }; } +function nextContinuousRunLabel(config, status) { + if (!config.enabled) return '-'; + if (status.running) return 'running now'; + const baseValue = status.last_iteration_at || status.last_finished_at || status.last_started_at; + if (!baseValue) return 'pending'; + const baseDate = new Date(baseValue); + if (Number.isNaN(baseDate.getTime())) return 'pending'; + const intervalMs = Math.max(1, Number(config.interval_minutes || 1)) * 60 * 1000; + const nextDate = new Date(baseDate.getTime() + intervalMs); + if (nextDate.getTime() <= Date.now()) return 'due now'; + return displayTime(nextDate.toISOString()); +} + function renderSummary(accountId, data) { const panel = document.getElementById(`panel-${accountId}`); if (!panel) return; const d = data.dashboard || data.state || {}; - panel.querySelector(".forwarding-count").textContent = String((d.forwarding_rules || []).length); - const toggle = panel.querySelector(".scrape-media-label"); - toggle.textContent = d.scrape_media ? "ON" : "OFF"; + const health = data.health || {}; + panel.querySelector('.forwarding-count').textContent = String((d.forwarding_rules || []).length); + const toggle = panel.querySelector('.scrape-media-label'); + toggle.textContent = d.scrape_media ? 'ON' : 'OFF'; + const healthOk = health.api_credentials && health.session_ready && health.data_dir_exists; + panel.querySelector('.account-health-label').textContent = healthOk ? 'Ready' : 'Check'; + panel.querySelector('.account-health-label').style.color = healthOk ? 'var(--ok)' : 'var(--warn)'; + panel.querySelector('.account-health-detail').textContent = [ + `${health.message_count || 0} messages`, + `${health.media_count || 0} media`, + health.active_job ? `active: ${health.active_job.status}` : 'idle', + ].join(' | '); } /* ── Account data loading ───────────────────────────── */ @@ -447,10 +651,10 @@ async function refreshAccount(accountId) { // Update tab status indicator const tab = document.querySelector(`.account-tab[data-account-id="${accountId}"]`); if (tab) { - const statusEl = tab.querySelector(".account-tab-status"); + const statusEl = tab.querySelector('.account-tab-status'); const authOk = isAccountAuthorized(authData); - statusEl.textContent = authOk ? "●" : "○"; - statusEl.className = "account-tab-status " + (authOk ? "ok" : ""); + statusEl.textContent = authOk ? '●' : '○'; + statusEl.className = 'account-tab-status ' + (authOk ? 'ok' : ''); } } catch (err) { console.error(`Failed to refresh account ${accountId}:`, err); @@ -460,14 +664,14 @@ async function refreshAccount(accountId) { async function loadAccounts() { let accounts = []; try { - const resp = await api("/api/accounts"); + const resp = await api('/api/accounts'); accounts = resp.accounts || []; } catch { // Legacy fallback — check /api/auth try { - const legacy = await api("/api/auth"); + const legacy = await api('/api/auth'); if (legacy.saved_credentials?.api_id) { - accounts = [{ id: "default", label: "Default", auth: legacy.auth_status || legacy }]; + accounts = [{ id: 'default', label: 'Default', auth: legacy.auth_status || legacy }]; } } catch { // No accounts at all @@ -477,11 +681,11 @@ async function loadAccounts() { state.accounts = accounts; if (accounts.length === 0) { - document.getElementById("no-accounts-msg").classList.remove("hidden"); + document.getElementById('no-accounts-msg').classList.remove('hidden'); return; } - document.getElementById("no-accounts-msg").classList.add("hidden"); + document.getElementById('no-accounts-msg').classList.add('hidden'); // Render tabs renderAccountTabs(accounts); @@ -490,9 +694,10 @@ async function loadAccounts() { accounts.forEach((acc) => renderAccountPanel(acc.id)); // Determine active account (persisted across reloads) - const savedId = localStorage.getItem("activeAccount"); - const activeId = (savedId && accounts.some(a => a.id === savedId)) ? savedId : (state.activeAccount || accounts[0].id); - switchAccount(activeId); + const activeId = loadSavedActiveAccount(accounts); + if (activeId) { + switchAccount(activeId); + } // Update settings renderSettingsAccounts(); @@ -501,36 +706,48 @@ async function loadAccounts() { /* ── Settings: Accounts list ────────────────────────── */ function renderSettingsAccounts() { - const container = document.getElementById("accounts-list"); - const template = document.getElementById("account-list-item-template"); - container.innerHTML = ""; + const container = document.getElementById('accounts-list'); + const template = document.getElementById('account-list-item-template'); + container.innerHTML = ''; state.accounts.forEach((acc) => { const node = template.content.firstElementChild.cloneNode(true); - node.querySelector(".account-list-label").textContent = acc.label || acc.id; - node.querySelector(".account-list-id").textContent = acc.id; + node.querySelector('.account-list-label').textContent = acc.label || acc.id; + node.querySelector('.account-list-id').textContent = acc.id; - const authStatus = node.querySelector(".account-list-auth-status"); + const authStatus = node.querySelector('.account-list-auth-status'); const authOk = isAccountAuthorized(acc.auth); - authStatus.textContent = authOk ? "Authorized" : "Needs auth"; - authStatus.style.color = authOk ? "var(--ok)" : "var(--danger)"; - authStatus.style.fontSize = "0.78rem"; + authStatus.textContent = authOk ? 'Authorized' : 'Needs auth'; + authStatus.style.color = authOk ? 'var(--ok)' : 'var(--danger)'; + authStatus.style.fontSize = '0.78rem'; - node.querySelector(".account-select-btn").addEventListener("click", () => { + node.querySelector('.account-select-btn').addEventListener('click', () => { switchAccount(acc.id); - document.getElementById("settings-dialog").close(); + document.getElementById('settings-dialog').close(); }); - node.querySelector(".account-remove-btn").addEventListener("click", async () => { + node.querySelector('.account-export-btn').addEventListener('click', async () => { + try { + const payload = await api(`/api/accounts/${acc.id}/export`); + downloadJson(`telegram-scraper-account-${acc.id}.json`, payload); + showToast(`Exported ${acc.label || acc.id}.`, 'success'); + } catch (err) { + showToast(`Failed to export account: ${err.message}`, 'error'); + } + }); + + node.querySelector('.account-remove-btn').addEventListener('click', async () => { if (!confirmAction(`Remove account "${acc.label || acc.id}"? All its data will be deleted.`)) return; try { - await api(`/api/accounts/${acc.id}`, { method: "DELETE" }); - await loadAccounts(); - if (state.activeAccount === acc.id) { + const removingActive = state.activeAccount === acc.id; + await api(`/api/accounts/${acc.id}`, { method: 'DELETE' }); + if (removingActive) { state.activeAccount = null; + localStorage.removeItem(ACTIVE_ACCOUNT_STORAGE_KEY); } + await loadAccounts(); } catch (err) { - alert(`Failed to remove account: ${err.message}`); + showToast(`Failed to remove account: ${err.message}`, 'error'); } }); @@ -541,17 +758,17 @@ function renderSettingsAccounts() { /* ── Auth (operates on active account) ───────────────── */ function updateAuthSection(accountId) { - const label = document.getElementById("auth-account-label"); + const label = document.getElementById('auth-account-label'); const acc = state.accounts.find((a) => a.id === accountId); - label.textContent = acc ? (acc.label || acc.id) : "-"; + label.textContent = acc ? acc.label || acc.id : '-'; } async function saveCredentials(accountId) { - const apiId = document.getElementById("api-id-input").value.trim(); - const apiHash = document.getElementById("api-hash-input").value.trim(); + const apiId = document.getElementById('api-id-input').value.trim(); + const apiHash = document.getElementById('api-hash-input').value.trim(); if (!apiId || !apiHash) return; await api(`/api/accounts/${accountId}/auth/credentials`, { - method: "POST", + method: 'POST', body: JSON.stringify({ api_id: apiId, api_hash: apiHash }), }); await loadAccounts(); @@ -559,45 +776,45 @@ async function saveCredentials(accountId) { async function startQrLogin(accountId) { const data = await api(`/api/accounts/${accountId}/auth/qr/start`, { - method: "POST", + method: 'POST', body: JSON.stringify({}), }); if (data.qr_image) { - document.getElementById("qr-image").src = data.qr_image; - document.getElementById("qr-wrap").classList.remove("hidden"); + document.getElementById('qr-image').src = data.qr_image; + document.getElementById('qr-wrap').classList.remove('hidden'); } await loadAccounts(); } async function requestPhoneCode(accountId) { - const phone = document.getElementById("phone-input").value.trim(); + const phone = document.getElementById('phone-input').value.trim(); if (!phone) return; await api(`/api/accounts/${accountId}/auth/phone/request`, { - method: "POST", + method: 'POST', body: JSON.stringify({ phone }), }); await loadAccounts(); } async function submitPhoneCode(accountId) { - const code = document.getElementById("code-input").value.trim(); + const code = document.getElementById('code-input').value.trim(); if (!code) return; await api(`/api/accounts/${accountId}/auth/phone/submit`, { - method: "POST", + method: 'POST', body: JSON.stringify({ code }), }); - document.getElementById("code-input").value = ""; + document.getElementById('code-input').value = ''; await loadAccounts(); } async function submitPassword(accountId) { - const password = document.getElementById("password-input").value.trim(); + const password = document.getElementById('password-input').value.trim(); if (!password) return; await api(`/api/accounts/${accountId}/auth/password`, { - method: "POST", + method: 'POST', body: JSON.stringify({ password }), }); - document.getElementById("password-input").value = ""; + document.getElementById('password-input').value = ''; await loadAccounts(); } @@ -605,9 +822,9 @@ async function submitPassword(accountId) { async function scrapeAll() { if (!state.activeAccount) return; - if (!confirmAction("Queue scraping for all tracked channels?")) return; + if (!confirmAction('Queue scraping for all tracked channels?')) return; await api(`/api/accounts/${state.activeAccount}/jobs/scrape`, { - method: "POST", + method: 'POST', body: JSON.stringify({}), }); await refreshAccount(state.activeAccount); @@ -615,9 +832,9 @@ async function scrapeAll() { async function exportAll() { if (!state.activeAccount) return; - if (!confirmAction("Queue export for all tracked channels?")) return; + if (!confirmAction('Queue export for all tracked channels?')) return; await api(`/api/accounts/${state.activeAccount}/jobs/export`, { - method: "POST", + method: 'POST', body: JSON.stringify({}), }); await refreshAccount(state.activeAccount); @@ -626,7 +843,7 @@ async function exportAll() { async function refreshDialogs() { if (!state.activeAccount) return; await api(`/api/accounts/${state.activeAccount}/jobs/refresh-dialogs`, { - method: "POST", + method: 'POST', body: JSON.stringify({}), }); await refreshAccount(state.activeAccount); @@ -635,7 +852,7 @@ async function refreshDialogs() { async function toggleMedia(value) { if (!state.activeAccount) return; await api(`/api/accounts/${state.activeAccount}/settings/media`, { - method: "POST", + method: 'POST', body: JSON.stringify({ value }), }); await refreshAccount(state.activeAccount); @@ -645,42 +862,60 @@ async function toggleMedia(value) { async function main() { // ── Sidebar action buttons ── - document.getElementById("scrape-all-btn").addEventListener("click", async () => { - try { await scrapeAll(); } catch (err) { alert("Scrape error: " + err.message); } + document.getElementById('scrape-all-btn').addEventListener('click', async () => { + try { + await scrapeAll(); + showToast('Scrape job queued.', 'success'); + } catch (err) { + showToast('Scrape error: ' + err.message, 'error'); + } }); - document.getElementById("export-all-btn").addEventListener("click", async () => { - try { await exportAll(); } catch (err) { alert("Export error: " + err.message); } + document.getElementById('export-all-btn').addEventListener('click', async () => { + try { + await exportAll(); + showToast('Export job queued.', 'success'); + } catch (err) { + showToast('Export error: ' + err.message, 'error'); + } }); - document.getElementById("refresh-dialogs-btn").addEventListener("click", async () => { - try { await refreshDialogs(); } catch (err) { alert("Dialog refresh error: " + err.message); } + document.getElementById('refresh-dialogs-btn').addEventListener('click', async () => { + try { + await refreshDialogs(); + showToast('Dialog refresh queued.', 'success'); + } catch (err) { + showToast('Dialog refresh error: ' + err.message, 'error'); + } }); // ── Settings dialog ── - const settingsDialog = document.getElementById("settings-dialog"); - document.getElementById("open-settings-btn").addEventListener("click", () => { - // Update auth section for active account - if (state.activeAccount) { - updateAuthSection(state.activeAccount); - } - renderSettingsAccounts(); - settingsDialog.showModal(); - }); - document.getElementById("close-settings-btn").addEventListener("click", () => { + const settingsDialog = document.getElementById('settings-dialog'); + const openSettingsBtn = document.getElementById('open-settings-btn'); + if (!openSettingsBtn.matches('a[href]')) { + openSettingsBtn.addEventListener('click', () => { + // Update auth section for active account + if (state.activeAccount) { + updateAuthSection(state.activeAccount); + } + renderSettingsAccounts(); + settingsDialog.showModal(); + }); + } + document.getElementById('close-settings-btn').addEventListener('click', () => { settingsDialog.close(); }); // ── Add account form ── - document.getElementById("add-account-form").addEventListener("submit", async (event) => { + document.getElementById('add-account-form').addEventListener('submit', async (event) => { event.preventDefault(); - const accountId = document.getElementById("add-account-id").value.trim(); - const label = document.getElementById("add-account-label").value.trim(); - const apiId = document.getElementById("add-account-api-id").value.trim(); - const apiHash = document.getElementById("add-account-api-hash").value.trim(); + const accountId = document.getElementById('add-account-id').value.trim(); + const label = document.getElementById('add-account-label').value.trim(); + const apiId = document.getElementById('add-account-api-id').value.trim(); + const apiHash = document.getElementById('add-account-api-hash').value.trim(); if (!accountId) return; try { - await api("/api/accounts", { - method: "POST", + await api('/api/accounts', { + method: 'POST', body: JSON.stringify({ account_id: accountId, label: label || accountId, @@ -694,71 +929,96 @@ async function main() { switchAccount(accountId); } } catch (err) { - alert("Failed to add account: " + err.message); + showToast('Failed to add account: ' + err.message, 'error'); + } + }); + + document.getElementById('import-account-file').addEventListener('change', async (event) => { + const file = event.currentTarget.files?.[0]; + if (!file) return; + try { + const payload = JSON.parse(await file.text()); + const accountId = payload.account_id || payload.id; + if (!accountId) throw new Error('account_id is missing in import file'); + await api('/api/accounts/import', { + method: 'POST', + body: JSON.stringify(payload), + }); + event.currentTarget.value = ''; + await loadAccounts(); + switchAccount(accountId); + showToast(`Imported ${accountId}.`, 'success'); + } catch (err) { + showToast(`Failed to import account: ${err.message}`, 'error'); } }); // ── Credentials form ── - document.getElementById("credentials-form").addEventListener("submit", async (event) => { + document.getElementById('credentials-form').addEventListener('submit', async (event) => { event.preventDefault(); if (!state.activeAccount) return; try { await saveCredentials(state.activeAccount); } catch (err) { - alert("Failed to save credentials: " + err.message); + showToast('Failed to save credentials: ' + err.message, 'error'); } }); // ── QR login ── - document.getElementById("start-qr-btn").addEventListener("click", async (event) => { + document.getElementById('start-qr-btn').addEventListener('click', async (event) => { setBusy(event.currentTarget, true); try { if (!state.activeAccount) return; await startQrLogin(state.activeAccount); } catch (err) { - alert("QR login error: " + err.message); + showToast('QR login error: ' + err.message, 'error'); } finally { setBusy(event.currentTarget, false); } }); // ── Phone ── - document.getElementById("phone-form").addEventListener("submit", async (event) => { + document.getElementById('phone-form').addEventListener('submit', async (event) => { event.preventDefault(); if (!state.activeAccount) return; try { await requestPhoneCode(state.activeAccount); } catch (err) { - alert("Phone code request error: " + err.message); + showToast('Phone code request error: ' + err.message, 'error'); } }); // ── Code ── - document.getElementById("code-form").addEventListener("submit", async (event) => { + document.getElementById('code-form').addEventListener('submit', async (event) => { event.preventDefault(); if (!state.activeAccount) return; try { await submitPhoneCode(state.activeAccount); } catch (err) { - alert("Code submit error: " + err.message); + showToast('Code submit error: ' + err.message, 'error'); } }); // ── Password ── - document.getElementById("password-form").addEventListener("submit", async (event) => { + document.getElementById('password-form').addEventListener('submit', async (event) => { event.preventDefault(); if (!state.activeAccount) return; try { await submitPassword(state.activeAccount); } catch (err) { - alert("Password error: " + err.message); + showToast('Password error: ' + err.message, 'error'); } }); // ── Media toggle ── - document.getElementById("scrape-media-toggle").addEventListener("change", async (event) => { + document.getElementById('scrape-media-toggle').addEventListener('change', async (event) => { const checked = event.currentTarget.checked; - try { await toggleMedia(checked); } catch (err) { alert("Media toggle error: " + err.message); } + try { + await toggleMedia(checked); + showToast('Media setting updated.', 'success'); + } catch (err) { + showToast('Media toggle error: ' + err.message, 'error'); + } }); // ── Load accounts ── @@ -774,5 +1034,5 @@ async function main() { main().catch((error) => { console.error(error); - alert(error.message); + showToast(error.message, 'error'); }); diff --git a/webui/index.html b/webui/index.html index b979a7c..689779c 100644 --- a/webui/index.html +++ b/webui/index.html @@ -1,10 +1,10 @@ - + Telegram Scraper Control Panel - +
@@ -17,6 +17,7 @@
+
+
Account health
+ + +
@@ -247,6 +259,7 @@
Status: -
Last run: -
+
Next run: -
Last error: -
@@ -300,6 +313,7 @@ diff --git a/webui/settings.html b/webui/settings.html new file mode 100644 index 0000000..6872341 --- /dev/null +++ b/webui/settings.html @@ -0,0 +1,156 @@ + + + + + + Telegram Scraper Settings + + + +
+ + +
+
+
+
Account Settings
+

Loading accounts

+

Select an account to manage credentials and scraper options.

+
+
+ + +
+
+ + + +
+
+
Credentials
+
-
+
+
+
Session
+
-
+
+
+
Continuous
+
-
+
+
+
Last scrape
+
-
+
+
+ +
+
+
+
+

Create or Import

+

New accounts start without continuous scraping enabled.

+
+
+
+ + + + + +
+ +
+ +
+
+
+

Credentials & Auth

+

No account selected.

+
+
+
+ + + +
+ +
+ +
+ + + +
+ + +
+ + + + +
+ +
+
+
+

Scraping

+

Account-level parser settings.

+
+
+ +
+
Tracked channels: 0
+
Messages: 0
+
Media: 0
+
+
+
+
+
+ + + + diff --git a/webui/settings.js b/webui/settings.js new file mode 100644 index 0000000..e654ecf --- /dev/null +++ b/webui/settings.js @@ -0,0 +1,431 @@ +const settingsState = { + accounts: [], + activeAccount: null, + dashboard: null, + auth: null, + continuous: null, +}; + +const ACTIVE_ACCOUNT_STORAGE_KEY = 'telegramScraper.activeAccount'; + +async function api(path, options = {}) { + const response = await fetch(path, { + headers: { 'Content-Type': 'application/json' }, + ...options, + }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.error || 'Request failed'); + } + return data; +} + +function showToast(message, type = 'info') { + let root = document.getElementById('toast-root'); + if (!root) { + root = document.createElement('div'); + root.id = 'toast-root'; + root.className = 'toast-root'; + document.body.appendChild(root); + } + const toast = document.createElement('div'); + toast.className = `toast toast-${type}`; + toast.textContent = message; + root.appendChild(toast); + window.setTimeout(() => { + toast.classList.add('toast-hide'); + window.setTimeout(() => toast.remove(), 250); + }, 3600); +} + +function downloadJson(filename, payload) { + const blob = new Blob([JSON.stringify(payload, null, 2) + '\n'], { + type: 'application/json', + }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); +} + +function displayTime(value) { + if (!value) return '-'; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return String(value).replace('T', ' ').replace('+00:00', ' UTC'); + return date.toLocaleString(undefined, { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }); +} + +function isAccountAuthorized(auth) { + const status = auth?.auth_status || auth || {}; + return ( + auth?.phase === 'authorized' || + auth?.status === 'authorized' || + status.status === 'ready' || + status.status === 'authorized' + ); +} + +function accountLabel(account) { + return account?.label || account?.id || 'No account'; +} + +function setActiveAccount(accountId) { + settingsState.activeAccount = accountId || null; + if (accountId) { + localStorage.setItem(ACTIVE_ACCOUNT_STORAGE_KEY, accountId); + } else { + localStorage.removeItem(ACTIVE_ACCOUNT_STORAGE_KEY); + } +} + +function chooseInitialAccount() { + const requested = new URLSearchParams(window.location.search).get('account'); + if (requested && settingsState.accounts.some((account) => account.id === requested)) { + return requested; + } + const saved = localStorage.getItem(ACTIVE_ACCOUNT_STORAGE_KEY); + if (saved && settingsState.accounts.some((account) => account.id === saved)) { + return saved; + } + return settingsState.accounts[0]?.id || null; +} + +function renderAccountList() { + const root = document.getElementById('settings-account-list'); + root.innerHTML = ''; + + if (!settingsState.accounts.length) { + root.innerHTML = '
No accounts yet.
'; + return; + } + + settingsState.accounts.forEach((account) => { + const button = document.createElement('button'); + button.className = 'settings-account-button'; + button.type = 'button'; + button.classList.toggle('active', account.id === settingsState.activeAccount); + + const name = document.createElement('span'); + name.className = 'settings-account-name'; + name.textContent = accountLabel(account); + + const status = document.createElement('span'); + status.className = isAccountAuthorized(account.auth) ? 'settings-account-status ok' : 'settings-account-status'; + status.textContent = isAccountAuthorized(account.auth) ? 'Authorized' : 'Needs auth'; + + button.append(name, status); + button.addEventListener('click', () => loadAccount(account.id)); + root.appendChild(button); + }); +} + +function renderEmptyState() { + const hasAccount = Boolean(settingsState.activeAccount); + document.getElementById('settings-empty').classList.toggle('hidden', hasAccount); + document.getElementById('settings-export-btn').disabled = !hasAccount; + document.getElementById('settings-delete-btn').disabled = !hasAccount; + document + .getElementById('settings-credentials-form') + .querySelectorAll('input, button') + .forEach((node) => { + node.disabled = !hasAccount; + }); + document.getElementById('settings-start-qr-btn').disabled = !hasAccount; + document + .getElementById('settings-phone-form') + .querySelectorAll('input, button') + .forEach((node) => { + node.disabled = !hasAccount; + }); + document.getElementById('settings-scrape-media-toggle').disabled = !hasAccount; +} + +function renderAccountData() { + const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount); + const dashboard = settingsState.dashboard || {}; + const health = dashboard.health || {}; + const auth = settingsState.auth || account?.auth || {}; + const continuous = settingsState.continuous || {}; + const continuousStatus = continuous.status || {}; + const continuousConfig = continuous.config || {}; + + document.getElementById('settings-title').textContent = account ? accountLabel(account) : 'No account selected'; + document.getElementById('settings-subtitle').textContent = account + ? `Account ID: ${account.id}` + : 'Create or import an account to manage settings.'; + document.getElementById('settings-auth-label').textContent = account + ? `${accountLabel(account)} auth status` + : 'No account selected.'; + + document.getElementById('health-credentials').textContent = health.api_credentials ? 'Saved' : 'Missing'; + document.getElementById('health-credentials').style.color = health.api_credentials ? 'var(--ok)' : 'var(--warn)'; + document.getElementById('health-session').textContent = + health.session_ready || isAccountAuthorized(auth) ? 'Ready' : 'Missing'; + document.getElementById('health-session').style.color = + health.session_ready || isAccountAuthorized(auth) ? 'var(--ok)' : 'var(--warn)'; + document.getElementById('health-continuous').textContent = continuousStatus.running + ? 'Running' + : continuousConfig.enabled + ? 'Enabled' + : 'Stopped'; + document.getElementById('health-continuous').style.color = continuousStatus.running ? 'var(--ok)' : 'var(--dim)'; + document.getElementById('health-last-scrape').textContent = displayTime( + health.last_scrape || continuousStatus.last_iteration_at, + ); + + document.getElementById('settings-scrape-media-toggle').checked = Boolean(dashboard.dashboard?.scrape_media); + document.getElementById('settings-channel-count').textContent = String(health.channel_count || 0); + document.getElementById('settings-message-count').textContent = String(health.message_count || 0); + document.getElementById('settings-media-count').textContent = String(health.media_count || 0); + renderEmptyState(); + renderAccountList(); +} + +async function loadAccounts() { + const payload = await api('/api/accounts'); + settingsState.accounts = payload.accounts || []; + setActiveAccount(chooseInitialAccount()); + renderAccountList(); + if (settingsState.activeAccount) { + await loadAccount(settingsState.activeAccount, { preserveUrl: true }); + } else { + settingsState.dashboard = null; + settingsState.auth = null; + settingsState.continuous = null; + renderAccountData(); + } +} + +async function loadAccount(accountId, options = {}) { + setActiveAccount(accountId); + const [dashboard, auth, continuous] = await Promise.all([ + api(`/api/accounts/${encodeURIComponent(accountId)}`), + api(`/api/accounts/${encodeURIComponent(accountId)}/auth`).catch(() => ({})), + api(`/api/accounts/${encodeURIComponent(accountId)}/continuous`).catch(() => ({})), + ]); + settingsState.dashboard = dashboard; + settingsState.auth = auth; + settingsState.continuous = continuous; + if (!options.preserveUrl) { + history.replaceState(null, '', `/settings?account=${encodeURIComponent(accountId)}`); + } + renderAccountData(); +} + +async function createAccount(event) { + event.preventDefault(); + const accountId = document.getElementById('settings-add-account-id').value.trim(); + const label = document.getElementById('settings-add-account-label').value.trim(); + const apiId = document.getElementById('settings-add-account-api-id').value.trim(); + const apiHash = document.getElementById('settings-add-account-api-hash').value.trim(); + if (!accountId) return; + + await api('/api/accounts', { + method: 'POST', + body: JSON.stringify({ + account_id: accountId, + label: label || accountId, + api_id: apiId ? parseInt(apiId) : undefined, + api_hash: apiHash || undefined, + }), + }); + event.currentTarget.reset(); + await loadAccounts(); + await loadAccount(accountId); + showToast(`Added ${accountId}.`, 'success'); +} + +async function importAccount(event) { + const file = event.currentTarget.files?.[0]; + if (!file) return; + try { + const payload = JSON.parse(await file.text()); + const accountId = payload.account_id || payload.id; + if (!accountId) throw new Error('account_id is missing in import file'); + await api('/api/accounts/import', { + method: 'POST', + body: JSON.stringify(payload), + }); + event.currentTarget.value = ''; + await loadAccounts(); + await loadAccount(accountId); + showToast(`Imported ${accountId}.`, 'success'); + } catch (err) { + showToast(`Failed to import account: ${err.message}`, 'error'); + } +} + +async function saveCredentials(event) { + event.preventDefault(); + if (!settingsState.activeAccount) return; + const apiId = document.getElementById('settings-api-id-input').value.trim(); + const apiHash = document.getElementById('settings-api-hash-input').value.trim(); + if (!apiId || !apiHash) return; + await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/credentials`, { + method: 'POST', + body: JSON.stringify({ api_id: apiId, api_hash: apiHash }), + }); + await loadAccount(settingsState.activeAccount); + showToast('Credentials saved.', 'success'); +} + +async function startQrLogin() { + if (!settingsState.activeAccount) return; + const data = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/qr/start`, { + method: 'POST', + body: JSON.stringify({}), + }); + if (data.qr_image) { + document.getElementById('settings-qr-image').src = data.qr_image; + document.getElementById('settings-qr-wrap').classList.remove('hidden'); + } + await loadAccount(settingsState.activeAccount); +} + +async function requestPhoneCode(event) { + event.preventDefault(); + if (!settingsState.activeAccount) return; + const phone = document.getElementById('settings-phone-input').value.trim(); + if (!phone) return; + await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/request`, { + method: 'POST', + body: JSON.stringify({ phone }), + }); + document.getElementById('settings-code-form').classList.remove('hidden'); + await loadAccount(settingsState.activeAccount); +} + +async function submitPhoneCode(event) { + event.preventDefault(); + if (!settingsState.activeAccount) return; + const code = document.getElementById('settings-code-input').value.trim(); + if (!code) return; + await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/submit`, { + method: 'POST', + body: JSON.stringify({ code }), + }); + document.getElementById('settings-code-input').value = ''; + await loadAccount(settingsState.activeAccount); +} + +async function submitPassword(event) { + event.preventDefault(); + if (!settingsState.activeAccount) return; + const password = document.getElementById('settings-password-input').value.trim(); + if (!password) return; + await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/password`, { + method: 'POST', + body: JSON.stringify({ password }), + }); + document.getElementById('settings-password-input').value = ''; + await loadAccount(settingsState.activeAccount); +} + +async function exportAccount() { + if (!settingsState.activeAccount) return; + const payload = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/export`); + downloadJson(`telegram-scraper-account-${settingsState.activeAccount}.json`, payload); + showToast('Account exported.', 'success'); +} + +async function deleteAccount() { + if (!settingsState.activeAccount) return; + const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount); + if (!window.confirm(`Remove account "${accountLabel(account)}"? All its account data will be deleted.`)) return; + await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}`, { method: 'DELETE' }); + setActiveAccount(null); + await loadAccounts(); + showToast('Account deleted.', 'success'); +} + +async function toggleMedia(event) { + if (!settingsState.activeAccount) return; + await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/settings/media`, { + method: 'POST', + body: JSON.stringify({ value: event.currentTarget.checked }), + }); + await loadAccount(settingsState.activeAccount); +} + +function bindEvents() { + document.getElementById('settings-add-account-form').addEventListener('submit', async (event) => { + try { + await createAccount(event); + } catch (err) { + showToast(`Failed to add account: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-import-account-file').addEventListener('change', importAccount); + document.getElementById('settings-credentials-form').addEventListener('submit', async (event) => { + try { + await saveCredentials(event); + } catch (err) { + showToast(`Failed to save credentials: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-start-qr-btn').addEventListener('click', async () => { + try { + await startQrLogin(); + } catch (err) { + showToast(`QR login failed: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-phone-form').addEventListener('submit', async (event) => { + try { + await requestPhoneCode(event); + } catch (err) { + showToast(`Phone login failed: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-code-form').addEventListener('submit', async (event) => { + try { + await submitPhoneCode(event); + } catch (err) { + showToast(`Code submit failed: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-password-form').addEventListener('submit', async (event) => { + try { + await submitPassword(event); + } catch (err) { + showToast(`Password submit failed: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-export-btn').addEventListener('click', async () => { + try { + await exportAccount(); + } catch (err) { + showToast(`Export failed: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-delete-btn').addEventListener('click', async () => { + try { + await deleteAccount(); + } catch (err) { + showToast(`Delete failed: ${err.message}`, 'error'); + } + }); + document.getElementById('settings-scrape-media-toggle').addEventListener('change', async (event) => { + try { + await toggleMedia(event); + } catch (err) { + showToast(`Media setting failed: ${err.message}`, 'error'); + } + }); +} + +bindEvents(); +loadAccounts().catch((err) => { + console.error(err); + showToast(`Failed to load settings: ${err.message}`, 'error'); +}); diff --git a/webui/style.css b/webui/style.css index 2e40e14..fe998f9 100644 --- a/webui/style.css +++ b/webui/style.css @@ -4,21 +4,24 @@ --panel: #0b0b0b; --panel-solid: #101010; --panel-alt: #050505; - --text-color: #f2f3f5; - --accent: #1fb9aa; - --accent-2: #58a6ff; - --bubble-own: #12342f; - --bubble-other: #171717; - --dim: #9aa0a6; - --line: #242424; + --text-color: #f3f5f8; + --accent: #2f57b8; + --accent-hover: #3b63c6; + --accent-soft: #0d1730; + --accent-line: #24428d; + --accent-2: #6f95eb; + --bubble-own: #0f1c3f; + --bubble-other: #111111; + --dim: #9ea7b7; + --line: #1f1f1f; --danger: #ff7373; --ok: #65e6a4; --warn: #f6c969; --info: #8ec7ff; - --radius: 18px; + --radius: 22px; --shadow: none; - --font-ui: "Segoe UI", "Helvetica Neue", Arial, sans-serif; - --font-mono: "IBM Plex Mono", "SFMono-Regular", Consolas, monospace; + --font-ui: 'Segoe UI', 'Helvetica Neue', Arial, sans-serif; + --font-mono: 'IBM Plex Mono', 'SFMono-Regular', Consolas, monospace; } * { @@ -27,7 +30,7 @@ body { margin: 0; - background: #000; + background: var(--bg-color); color: var(--text-color); font-family: var(--font-ui); font-size: 15px; @@ -42,8 +45,8 @@ a { a:hover, button:hover { - border-color: #333; - background: #151515; + border-color: #2b2b2b; + background: #111111; color: var(--text-color); } @@ -58,36 +61,48 @@ button { height: 100vh; overflow: hidden; display: grid; - grid-template-columns: 308px minmax(0, 1fr); + grid-template-columns: 360px minmax(0, 1fr); } .sidebar { - background: #050505; + background: var(--panel-alt); border-right: 1px solid var(--line); - padding: 24px 20px; + padding: 30px 24px; + overflow-y: auto; } .content { overflow-y: auto; - padding: 24px; + padding: 32px; max-width: 1440px; width: 100%; } +.brand-block { + padding: 0 2px; +} + .brand-block h1, .dialog-header h2 { margin: 0; color: var(--accent); - font-size: 1.5rem; + font-size: 1.65rem; letter-spacing: -0.04em; } +.brand-block .muted { + margin: 22px 0 0; + max-width: 26ch; + font-size: 1.03rem; + line-height: 1.62; +} + .eyebrow, .section-title { color: var(--dim); font-size: 0.75rem; text-transform: uppercase; - letter-spacing: 0.08em; + letter-spacing: 0.14em; } .muted { @@ -106,7 +121,7 @@ button { } .nav-links { - margin: 28px 0; + margin: 34px 0; } .nav-link, @@ -120,6 +135,12 @@ input, border-radius: 999px; } +a.nav-link, +a.button, +a.content-tab { + border-bottom: 1px solid var(--line); +} + .nav-link, .button, .content-tab { @@ -127,24 +148,33 @@ input, align-items: center; justify-content: center; min-height: 42px; - padding: 9px 13px; + padding: 10px 18px; cursor: pointer; - transition: background-color 0.18s ease, border-color 0.18s ease, transform 0.18s ease; + transition: + background-color 0.18s ease, + border-color 0.18s ease, + box-shadow 0.18s ease, + transform 0.18s ease; +} + +.nav-link { + min-height: 52px; + font-size: 1rem; } .nav-link.active, .button.primary, .content-tab.active { - border-color: #0f6f67; + border-color: var(--accent-line); color: #ffffff; - background: #0f6f67; + background: var(--accent); box-shadow: none; } .nav-link.active:hover, .button.primary:hover, .content-tab.active:hover { - background: #128277; + background: var(--accent-hover); color: #ffffff; } @@ -179,24 +209,35 @@ input, .panel, .job-card, .message-card { - padding: 16px; + padding: 22px; } .status-card { - margin-top: 16px; + margin-top: 20px; + display: grid; + gap: 12px; + align-content: start; } .status-card .button { width: 100%; - margin-top: 8px; + margin-top: 0; +} + +.status-card .section-title, +.status-card p, +.status-card .status-badge { + margin-top: 0; + margin-bottom: 0; } .status-badge { display: inline-flex; - padding: 6px 10px; + padding: 7px 12px; border: 1px solid var(--line); border-radius: 999px; color: var(--ok); + background: #0d1410; } .content-tabs { @@ -216,13 +257,17 @@ input, .panel-grid { display: grid; - gap: 16px; + gap: 18px; grid-template-columns: repeat(3, minmax(0, 1fr)); - margin-bottom: 16px; + margin-bottom: 18px; +} + +.settings-health-grid { + grid-template-columns: repeat(4, minmax(0, 1fr)); } .panel { - margin-bottom: 16px; + margin-bottom: 18px; } .panel-header, @@ -235,7 +280,7 @@ input, } .stat-value { - margin-top: 8px; + margin-top: 10px; font-size: 2.25rem; font-weight: 700; letter-spacing: -0.05em; @@ -257,15 +302,88 @@ input, margin-top: 12px; } +.settings-layout { + display: grid; + grid-template-columns: minmax(280px, 0.9fr) minmax(320px, 1.1fr); + gap: 18px; + align-items: start; +} + +.settings-layout .panel:last-child { + grid-column: 1 / -1; +} + +.settings-page-shell .content { + max-width: 1280px; +} + +.settings-hero { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 16px; +} + +.settings-account-list { + display: grid; + gap: 10px; + margin-top: 14px; +} + +.settings-account-button { + width: 100%; + min-height: 48px; + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; + align-items: center; + text-align: left; + border: 1px solid var(--line); + border-radius: 16px; + background: #101010; + color: var(--text-color); + padding: 12px 14px; + cursor: pointer; +} + +.settings-account-button.active { + border-color: var(--accent-line); + background: var(--accent-soft); +} + +.settings-account-name { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.settings-account-status { + color: var(--warn); + font-size: 0.78rem; +} + +.settings-account-status.ok { + color: var(--ok); +} + +.settings-meta-list { + display: grid; + gap: 8px; + margin-top: 16px; + color: var(--dim); +} + input { min-width: 160px; - padding: 10px 13px; + padding: 11px 14px; outline: 0; - border-radius: 14px; + border-radius: 16px; } input:focus { border-color: var(--accent); + box-shadow: none; } .toggle-row { @@ -301,7 +419,7 @@ input:focus { } .switch-slider::before { - content: ""; + content: ''; position: absolute; top: 4px; left: 4px; @@ -309,11 +427,14 @@ input:focus { height: 16px; background: var(--dim); border-radius: 50%; - transition: transform 0.16s ease, background-color 0.16s ease; + transition: + transform 0.16s ease, + background-color 0.16s ease; } .switch input:checked + .switch-slider { border-color: var(--accent); + background: var(--accent-soft); } .switch input:checked + .switch-slider::before { @@ -339,7 +460,7 @@ td { } tbody tr:hover { - background: #111; + background: #0d0d0d; } .action-row { @@ -386,10 +507,10 @@ tbody tr:hover { align-items: center; gap: 10px; min-height: 38px; - padding: 8px; + padding: 10px 12px; border: 1px solid var(--line); - border-radius: 14px; - background: #0f0f0f; + border-radius: 16px; + background: #101010; } .checkbox-row input { @@ -419,7 +540,7 @@ tbody tr:hover { } .log-line:hover { - background: #111; + background: rgba(53, 95, 199, 0.08); } .log-time, @@ -460,11 +581,11 @@ tbody tr:hover { } .settings-dialog::backdrop { - background: rgba(0, 0, 0, 0.72); + background: rgba(0, 0, 0, 0.82); } .dialog-shell { - padding: 18px; + padding: 24px; } .settings-section { @@ -503,12 +624,12 @@ tbody tr:hover { } .viewer-sidebar { - background: #050505; + background: var(--panel-alt); border-right: 1px solid var(--line); - padding: 14px 10px; + padding: 18px 14px; display: flex; flex-direction: column; - gap: 12px; + gap: 14px; overflow-y: auto; } @@ -527,13 +648,13 @@ tbody tr:hover { .viewer-sidebar-head h1 { margin: 0; color: var(--accent); - font-size: 1.45rem; + font-size: 1.55rem; letter-spacing: -0.04em; } .viewer-channel-list { display: grid; - gap: 2px; + gap: 6px; } .viewer-channel-item { @@ -543,23 +664,23 @@ tbody tr:hover { gap: 10px; width: 100%; min-height: 68px; - padding: 8px 10px; + padding: 10px 12px; text-align: left; cursor: pointer; border: 1px solid transparent; background: transparent; color: var(--text-color); - border-radius: 12px; + border-radius: 16px; } .viewer-channel-item.active { - border-color: transparent; - background: #12342f; + border-color: var(--accent-line); + background: var(--accent-soft); box-shadow: none; } .viewer-channel-item:hover { - background: #111; + background: #0f0f0f; } .viewer-channel-avatar { @@ -569,7 +690,7 @@ tbody tr:hover { width: 48px; height: 48px; border-radius: 50%; - background: #5f6b70; + background: #274894; color: white; font-weight: 700; letter-spacing: -0.03em; @@ -613,8 +734,8 @@ tbody tr:hover { min-width: 22px; padding: 2px 7px; border-radius: 999px; - background: #12342f; - color: var(--accent); + background: #111a32; + color: var(--accent-2); font-size: 0.76rem; text-align: center; } @@ -626,31 +747,56 @@ tbody tr:hover { flex-direction: column; height: 100vh; overflow: hidden; - background: - radial-gradient(circle at 20px 20px, rgba(255, 255, 255, 0.025) 1px, transparent 1.5px), - radial-gradient(circle at 58px 54px, rgba(255, 255, 255, 0.018) 1px, transparent 1.5px), - #000; - background-size: auto, 84px 84px, 84px 84px, auto; + background: var(--bg-color); } .viewer-header { flex-shrink: 0; - padding: 16px 24px; + padding: 18px 26px; border-bottom: 1px solid var(--line); - background: #050505; + background: var(--panel-alt); + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 16px; } .viewer-header h2 { margin: 0; color: var(--accent); - font-size: 1.45rem; + font-size: 1.55rem; letter-spacing: -0.04em; } +.viewer-tools { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; + flex-wrap: wrap; +} + +.viewer-tools input[type='search'] { + min-width: min(260px, 40vw); +} + +.viewer-auto-refresh { + display: inline-flex; + align-items: center; + gap: 6px; + min-height: 34px; + color: var(--dim); + font-size: 0.82rem; +} + +.viewer-auto-refresh input { + min-width: auto; +} + .messages-list { flex: 1; overflow-y: auto; - padding: 18px 34px 22px; + padding: 22px 38px 26px; display: flex; flex-direction: column; } @@ -676,7 +822,7 @@ tbody tr:hover { .chat-date-sep::before, .chat-date-sep::after { - content: ""; + content: ''; flex: 1; height: 1px; background: #242424; @@ -686,7 +832,7 @@ tbody tr:hover { align-self: center; padding: 5px 12px; border-radius: 999px; - background: #111; + background: #111111; } .chat-date-sep:empty { @@ -709,11 +855,11 @@ tbody tr:hover { .chat-bubble { max-width: min(680px, 74%); - padding: 8px 12px 7px; + padding: 9px 13px 8px; background: var(--bubble-other); - border: 1px solid #242424; + border: 1px solid var(--line); border-radius: 18px 18px 18px 6px; - margin-bottom: 6px; + margin-bottom: 7px; overflow-wrap: anywhere; word-break: break-word; position: relative; @@ -721,7 +867,7 @@ tbody tr:hover { } .chat-bubble::before { - content: ""; + content: ''; position: absolute; top: 8px; border: 6px solid transparent; @@ -741,7 +887,7 @@ tbody tr:hover { .chat-bubble-wrap--own .chat-bubble { background: var(--bubble-own); - border-color: #1f5b52; + border-color: var(--accent-line); border-radius: 18px 18px 6px 18px; } @@ -750,8 +896,13 @@ tbody tr:hover { } @keyframes chat-highlight { - 0%, 100% { box-shadow: 0 0 0 0 rgba(139, 180, 248, 0); } - 50% { box-shadow: 0 0 0 3px rgba(139, 180, 248, 0.4); } + 0%, + 100% { + box-shadow: 0 0 0 0 rgba(139, 180, 248, 0); + } + 50% { + box-shadow: 0 0 0 3px rgba(139, 180, 248, 0.4); + } } /* ---------- Sender ---------- */ @@ -852,6 +1003,55 @@ tbody tr:hover { display: none; } +.empty-state, +.viewer-empty-state { + padding: 16px; + color: var(--dim); + border: 1px solid var(--line); + border-radius: 14px; + background: #0b0b0b; +} + +.viewer-empty-state { + align-self: center; + margin-top: 16px; +} + +.toast-root { + position: fixed; + right: 18px; + bottom: 18px; + z-index: 1000; + display: grid; + gap: 8px; + width: min(360px, calc(100vw - 36px)); +} + +.toast { + padding: 12px 14px; + border: 1px solid var(--line); + border-radius: 14px; + background: var(--panel-solid); + color: var(--text-color); + transition: + opacity 0.2s ease, + transform 0.2s ease; +} + +.toast-success { + border-color: var(--accent-line); +} + +.toast-error { + border-color: #7a2a2a; + color: #ffd4d4; +} + +.toast-hide { + opacity: 0; + transform: translateY(8px); +} + .api-docs { display: grid; gap: 16px; @@ -927,9 +1127,9 @@ tbody tr:hover { .account-tabs { display: flex; flex-wrap: wrap; - gap: 8px; - margin-bottom: 16px; - padding-bottom: 10px; + gap: 10px; + margin-bottom: 18px; + padding-bottom: 14px; border-bottom: 1px solid var(--line); } @@ -937,8 +1137,8 @@ tbody tr:hover { display: inline-flex; align-items: center; gap: 8px; - min-height: 38px; - padding: 7px 14px; + min-height: 40px; + padding: 8px 15px; border: 1px solid var(--line); border-radius: 999px; background: #101010; @@ -949,9 +1149,10 @@ tbody tr:hover { } .account-tab.active { - border-color: #0f6f67; + border-color: var(--accent-line); color: #ffffff; - background: #0f6f67; + background: var(--accent); + box-shadow: none; } .account-tab .account-tab-status { @@ -981,8 +1182,8 @@ tbody tr:hover { .accounts-list { display: grid; - gap: 8px; - margin-bottom: 12px; + gap: 10px; + margin-bottom: 14px; } .account-list-item { @@ -990,10 +1191,10 @@ tbody tr:hover { align-items: center; justify-content: space-between; gap: 12px; - padding: 10px 12px; + padding: 12px 14px; border: 1px solid var(--line); border-radius: 16px; - background: #0f0f0f; + background: #101010; } .account-list-info { @@ -1022,12 +1223,22 @@ tbody tr:hover { .add-account-form { display: grid; - gap: 8px; - margin-top: 8px; - padding: 12px; + gap: 10px; + margin-top: 10px; + padding: 14px; border: 1px solid var(--line); - border-radius: 16px; - background: #0f0f0f; + border-radius: 18px; + background: #101010; +} + +.import-account-row { + display: grid; + gap: 8px; + margin-top: 12px; + padding: 14px; + border: 1px solid var(--line); + border-radius: 18px; + background: #101010; } /* ── Active Account Card ─────────────────────────────── */ @@ -1064,7 +1275,7 @@ tbody tr:hover { } .viewer-sidebar.open::before { - content: ""; + content: ''; position: fixed; inset: 0; background: rgba(0, 0, 0, 0.5); diff --git a/webui/swagger.html b/webui/swagger.html index 9c6c3cd..a4dddf1 100644 --- a/webui/swagger.html +++ b/webui/swagger.html @@ -1,10 +1,10 @@ - + Telegram Scraper API - +
diff --git a/webui/swagger.js b/webui/swagger.js index 3e64962..47f5d7b 100644 --- a/webui/swagger.js +++ b/webui/swagger.js @@ -1,37 +1,37 @@ async function loadSpec() { - const response = await fetch("/openapi.json"); + const response = await fetch('/openapi.json'); const spec = await response.json(); if (!response.ok) { - throw new Error(spec.error || "Failed to load OpenAPI spec"); + throw new Error(spec.error || 'Failed to load OpenAPI spec'); } return spec; } function methodPayload(operation) { - const body = operation.requestBody?.content?.["application/json"]?.schema; - if (!body) return ""; + const body = operation.requestBody?.content?.['application/json']?.schema; + if (!body) return ''; return JSON.stringify(body.example || body.properties || body, null, 2); } function renderSpec(spec) { - const root = document.getElementById("api-docs"); - const sectionTemplate = document.getElementById("api-section-template"); - const methodTemplate = document.getElementById("api-method-template"); - root.innerHTML = ""; + const root = document.getElementById('api-docs'); + const sectionTemplate = document.getElementById('api-section-template'); + const methodTemplate = document.getElementById('api-method-template'); + root.innerHTML = ''; Object.entries(spec.paths || {}).forEach(([path, methods]) => { const section = sectionTemplate.content.firstElementChild.cloneNode(true); - section.querySelector(".api-path").textContent = path; - const methodsRoot = section.querySelector(".api-methods"); + section.querySelector('.api-path').textContent = path; + const methodsRoot = section.querySelector('.api-methods'); Object.entries(methods).forEach(([method, operation]) => { const node = methodTemplate.content.firstElementChild.cloneNode(true); - node.querySelector(".api-verb").textContent = method.toUpperCase(); - node.querySelector(".api-summary").textContent = operation.summary || ""; - node.querySelector(".api-description").textContent = operation.description || ""; + node.querySelector('.api-verb').textContent = method.toUpperCase(); + node.querySelector('.api-summary').textContent = operation.summary || ''; + node.querySelector('.api-description').textContent = operation.description || ''; const body = methodPayload(operation); - const bodyNode = node.querySelector(".api-body"); + const bodyNode = node.querySelector('.api-body'); if (body) { bodyNode.textContent = body; } else { @@ -49,5 +49,5 @@ loadSpec() .then(renderSpec) .catch((error) => { console.error(error); - document.getElementById("api-docs").innerHTML = `
${error.message}
`; + document.getElementById('api-docs').innerHTML = `
${error.message}
`; }); diff --git a/webui/viewer.html b/webui/viewer.html index 96eae71..8e42d2f 100644 --- a/webui/viewer.html +++ b/webui/viewer.html @@ -1,10 +1,10 @@ - + Telegram Scraper Viewer - +
@@ -13,7 +13,10 @@
Export Viewer

Messages

-

Account:

+

+ Account: + +

Dashboard @@ -27,6 +30,14 @@

Select a channel

Reading messages from the local SQLite database.

+
+ + + +
diff --git a/webui/viewer.js b/webui/viewer.js index ae20025..903fade 100644 --- a/webui/viewer.js +++ b/webui/viewer.js @@ -1,4 +1,4 @@ -if ("scrollRestoration" in history) history.scrollRestoration = "manual"; +if ('scrollRestoration' in history) history.scrollRestoration = 'manual'; const viewerState = { accountId: null, @@ -8,7 +8,9 @@ const viewerState = { oldestMessageId: null, newestMessageId: null, userId: null, + search: '', loading: false, + autoRefreshTimer: null, sentinelObserver: null, }; @@ -16,98 +18,139 @@ async function api(path) { const response = await fetch(path); const data = await response.json(); if (!response.ok) { - throw new Error(data.error || "Request failed"); + throw new Error(data.error || 'Request failed'); } return data; } +function showToast(message, type = 'info') { + let root = document.getElementById('toast-root'); + if (!root) { + root = document.createElement('div'); + root.id = 'toast-root'; + root.className = 'toast-root'; + document.body.appendChild(root); + } + const toast = document.createElement('div'); + toast.className = `toast toast-${type}`; + toast.textContent = message; + root.appendChild(toast); + window.setTimeout(() => { + toast.classList.add('toast-hide'); + window.setTimeout(() => toast.remove(), 250); + }, 3600); +} + function formatDateHeader(dateStr) { - if (!dateStr) return ""; - const d = new Date(dateStr.replace(" ", "T")); - if (isNaN(d.getTime())) return ""; + if (!dateStr) return ''; + const d = new Date(dateStr.replace(' ', 'T')); + if (isNaN(d.getTime())) return ''; const now = new Date(); const diff = now - d; const oneDay = 86400000; - if (diff < oneDay && d.getDate() === now.getDate()) return "Today"; - if (diff < 2 * oneDay && d.getDate() === now.getDate() - 1) return "Yesterday"; - return d.toLocaleDateString("en-US", { - month: "long", - day: "numeric", - year: d.getFullYear() !== now.getFullYear() ? "numeric" : undefined, + if (diff < oneDay && d.getDate() === now.getDate()) return 'Today'; + if (diff < 2 * oneDay && d.getDate() === now.getDate() - 1) return 'Yesterday'; + return d.toLocaleDateString('en-US', { + month: 'long', + day: 'numeric', + year: d.getFullYear() !== now.getFullYear() ? 'numeric' : undefined, }); } function formatTime(dateStr) { - if (!dateStr) return ""; - const d = new Date(dateStr.replace(" ", "T")); - if (isNaN(d.getTime())) return ""; - return d.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit" }); + if (!dateStr) return ''; + const d = new Date(dateStr.replace(' ', 'T')); + if (isNaN(d.getTime())) return ''; + return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' }); } function chatListTime(dateStr) { - if (!dateStr) return ""; - const d = new Date(dateStr.replace(" ", "T")); - if (isNaN(d.getTime())) return ""; + if (!dateStr) return ''; + const d = new Date(dateStr.replace(' ', 'T')); + if (isNaN(d.getTime())) return ''; const now = new Date(); if (d.toDateString() === now.toDateString()) { - return d.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit" }); + return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' }); } - return d.toLocaleDateString("en-US", { month: "short", day: "numeric" }); + return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); } function initials(value) { - const source = String(value || "?").replace(/^@/, "").trim(); + const source = String(value || '?') + .replace(/^@/, '') + .trim(); const words = source.split(/\s+/).filter(Boolean); if (words.length > 1) return (words[0][0] + words[1][0]).toUpperCase(); return source.slice(0, 2).toUpperCase(); } function dateKey(dateStr) { - if (!dateStr) return ""; + if (!dateStr) return ''; return dateStr.slice(0, 10); } function textNodeWithBreaks(text) { const fragment = document.createDocumentFragment(); if (!text) return fragment; - const parts = text.split("\n"); + const parts = text.split('\n'); parts.forEach((part, index) => { - if (index > 0) fragment.appendChild(document.createElement("br")); + if (index > 0) fragment.appendChild(document.createElement('br')); fragment.appendChild(document.createTextNode(part)); }); return fragment; } function scrollToBottom() { - const list = document.getElementById("messages-list"); + const list = document.getElementById('messages-list'); if (!list) return; - const go = () => { list.scrollTop = list.scrollHeight; }; + const go = () => { + list.scrollTop = list.scrollHeight; + }; go(); requestAnimationFrame(go); setTimeout(go, 150); } +function makeScrollSentinel() { + const sentinel = document.createElement('div'); + sentinel.id = 'scroll-sentinel'; + sentinel.className = 'scroll-sentinel'; + return sentinel; +} + +function resetMessageView(title = 'Select a channel', subtitle = 'Reading messages from the local SQLite database.') { + const root = document.getElementById('messages-list'); + if (root) { + root.innerHTML = ''; + root.appendChild(makeScrollSentinel()); + } + document.getElementById('viewer-title').textContent = title; + document.getElementById('viewer-subtitle').textContent = subtitle; + viewerState.oldestMessageId = null; + viewerState.newestMessageId = null; +} + function renderChannelList() { - const root = document.getElementById("viewer-channel-list"); + const root = document.getElementById('viewer-channel-list'); if (!root) return; - root.innerHTML = ""; - const template = document.getElementById("viewer-channel-template"); + root.innerHTML = ''; + const template = document.getElementById('viewer-channel-template'); if (!template) return; viewerState.channels.forEach((channel) => { const node = template.content.firstElementChild.cloneNode(true); const preview = channel.last_message_preview || - (channel.has_database ? "No text in the last saved message" : "No local database yet"); - node.querySelector(".viewer-channel-avatar").textContent = initials(channel.name || channel.channel_id); - node.querySelector(".viewer-channel-name").textContent = channel.name; - node.querySelector(".viewer-channel-time").textContent = chatListTime(channel.last_date); - node.querySelector(".viewer-channel-preview").textContent = preview; - node.querySelector(".viewer-channel-count").textContent = String(channel.message_count || 0); + (channel.has_database ? 'No text in the last saved message' : 'No local database yet'); + node.querySelector('.viewer-channel-avatar').textContent = initials(channel.name || channel.channel_id); + node.querySelector('.viewer-channel-name').textContent = channel.name; + node.querySelector('.viewer-channel-time').textContent = chatListTime(channel.last_date); + node.querySelector('.viewer-channel-preview').textContent = preview; + node.querySelector('.viewer-channel-count').textContent = String(channel.message_count || 0); if (channel.channel_id === viewerState.channelId) { - node.classList.add("active"); + node.classList.add('active'); } - node.addEventListener("click", () => { - document.querySelector(".viewer-sidebar")?.classList.remove("open"); + node.addEventListener('click', () => { + document.querySelector('.viewer-sidebar')?.classList.remove('open'); viewerState.oldestMessageId = null; viewerState.newestMessageId = null; loadChannel(channel.channel_id); @@ -116,33 +159,34 @@ function renderChannelList() { }); } -function messageEndpoint(channelId, before = "") { +function messageEndpoint(channelId, before = '') { + const search = viewerState.search ? `&search=${encodeURIComponent(viewerState.search)}` : ''; if (viewerState.accountId) { - return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`; + return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`; } - return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}`; + return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`; } function channelsEndpoint() { if (viewerState.accountId) { return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels`; } - return "/api/channels"; + return '/api/channels'; } function renderAccountSelector() { - const sel = document.getElementById("viewer-account-select"); + const sel = document.getElementById('viewer-account-select'); if (!sel) return; - sel.innerHTML = ""; + sel.innerHTML = ''; const legacy = viewerState.accounts.length === 0; if (legacy) { - const opt = document.createElement("option"); - opt.value = ""; - opt.textContent = "Legacy"; + const opt = document.createElement('option'); + opt.value = ''; + opt.textContent = 'Legacy'; sel.appendChild(opt); } else { viewerState.accounts.forEach((acc) => { - const opt = document.createElement("option"); + const opt = document.createElement('option'); opt.value = acc.id; opt.textContent = acc.label || acc.id; if (acc.id === viewerState.accountId) opt.selected = true; @@ -153,9 +197,12 @@ function renderAccountSelector() { async function switchViewerAccount(accountId) { viewerState.accountId = accountId || null; + viewerState.search = ''; + const searchInput = document.getElementById('viewer-search'); + if (searchInput) searchInput.value = ''; viewerState.accounts.forEach((acc) => { if (acc.id === accountId) { - const opts = document.getElementById("viewer-account-select")?.options; + const opts = document.getElementById('viewer-account-select')?.options; if (opts) { for (let i = 0; i < opts.length; i++) { opts[i].selected = opts[i].value === accountId; @@ -166,10 +213,10 @@ async function switchViewerAccount(accountId) { try { const authData = viewerState.accountId ? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`) - : await api("/api/auth"); + : await api('/api/auth'); viewerState.userId = authData.user_id || null; } catch (e) { - console.warn("Could not fetch user ID:", e); + console.warn('Could not fetch user ID:', e); } const channels = await api(channelsEndpoint()); viewerState.channels = channels; @@ -179,94 +226,95 @@ async function switchViewerAccount(accountId) { renderChannelList(); if (viewerState.channelId) { await loadChannel(viewerState.channelId); + } else { + resetMessageView( + 'No channels', + viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.', + ); } } async function loadViewerAccount(params) { - const requested = params.get("account"); + const requested = params.get('account'); try { - const payload = await api("/api/accounts"); + const payload = await api('/api/accounts'); viewerState.accounts = payload.accounts || []; } catch { viewerState.accounts = []; } - viewerState.accountId = - requested || - viewerState.accounts[0]?.id || - null; + const requestedExists = requested && viewerState.accounts.some((acc) => acc.id === requested); + viewerState.accountId = requestedExists ? requested : viewerState.accounts[0]?.id || null; renderAccountSelector(); - const sel = document.getElementById("viewer-account-select"); + const sel = document.getElementById('viewer-account-select'); if (sel) { - sel.addEventListener("change", () => switchViewerAccount(sel.value)); + sel.addEventListener('change', () => switchViewerAccount(sel.value)); } try { const authData = viewerState.accountId ? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`) - : await api("/api/auth"); + : await api('/api/auth'); viewerState.userId = authData.user_id || null; } catch (e) { - console.warn("Could not fetch user ID:", e); + console.warn('Could not fetch user ID:', e); } } function buildMessageNode(message, root) { - const template = document.getElementById("message-template"); + const template = document.getElementById('message-template'); const wrap = template.content.firstElementChild.cloneNode(true); - const bubble = wrap.querySelector(".chat-bubble"); + const bubble = wrap.querySelector('.chat-bubble'); bubble.dataset.messageId = String(message.message_id); const isOwn = viewerState.userId && message.sender_id === viewerState.userId; if (isOwn) { - wrap.classList.add("chat-bubble-wrap--own"); + wrap.classList.add('chat-bubble-wrap--own'); } - const sender = bubble.querySelector(".chat-sender"); - sender.textContent = message.sender_name || ""; + const sender = bubble.querySelector('.chat-sender'); + sender.textContent = message.sender_name || ''; - const replyEl = bubble.querySelector(".chat-reply"); + const replyEl = bubble.querySelector('.chat-reply'); const replyMsg = message.reply_to_message; if (message.reply_to) { - replyEl.classList.remove("hidden"); - bubble.querySelector(".chat-reply-author").textContent = - replyMsg?.sender_name || `#${message.reply_to}`; - bubble.querySelector(".chat-reply-text").textContent = - replyMsg?.text || "(message not available)"; - replyEl.addEventListener("click", () => { + replyEl.classList.remove('hidden'); + bubble.querySelector('.chat-reply-author').textContent = replyMsg?.sender_name || `#${message.reply_to}`; + bubble.querySelector('.chat-reply-text').textContent = replyMsg?.text || '(message not available)'; + replyEl.addEventListener('click', () => { const target = root.querySelector(`[data-message-id="${message.reply_to}"]`); if (target) { - target.scrollIntoView({ behavior: "smooth", block: "center" }); - target.classList.add("chat-bubble--highlight"); - setTimeout(() => target.classList.remove("chat-bubble--highlight"), 2000); + target.scrollIntoView({ behavior: 'smooth', block: 'center' }); + target.classList.add('chat-bubble--highlight'); + setTimeout(() => target.classList.remove('chat-bubble--highlight'), 2000); } }); } else { - replyEl.classList.add("hidden"); + replyEl.classList.add('hidden'); } - bubble.querySelector(".chat-text").appendChild(textNodeWithBreaks(message.text || "")); + bubble.querySelector('.chat-text').appendChild(textNodeWithBreaks(message.text || '')); - const media = bubble.querySelector(".chat-media"); - if (message.media_url && message.media_kind === "image") { - const img = document.createElement("img"); + const media = bubble.querySelector('.chat-media'); + if (message.media_url && message.media_kind === 'image') { + const img = document.createElement('img'); img.src = message.media_url; - img.loading = "lazy"; + img.loading = 'lazy'; media.appendChild(img); - } else if (message.media_url && message.media_kind === "video") { - const video = document.createElement("video"); + } else if (message.media_url && message.media_kind === 'video') { + const video = document.createElement('video'); video.src = message.media_url; video.controls = true; - video.preload = "metadata"; + video.preload = 'metadata'; media.appendChild(video); } else if (message.media_url) { - const link = document.createElement("a"); + const link = document.createElement('a'); link.href = message.media_url; - link.target = "_blank"; - link.rel = "noreferrer"; - link.textContent = "Open file"; + link.target = '_blank'; + link.rel = 'noreferrer'; + link.textContent = 'Open file'; media.appendChild(link); } @@ -275,40 +323,40 @@ function buildMessageNode(message, root) { if (message.views) footerParts.push(`views: ${message.views}`); if (message.forwards) footerParts.push(`forwards: ${message.forwards}`); if (message.reactions) footerParts.push(`reactions: ${message.reactions}`); - bubble.querySelector(".chat-footer").textContent = footerParts.join(" | "); + bubble.querySelector('.chat-footer').textContent = footerParts.join(' | '); return { wrap, dateKey: dateKey(message.date) }; } function makeDateSep(text) { - const el = document.createElement("div"); - el.className = "chat-date-sep"; + const el = document.createElement('div'); + el.className = 'chat-date-sep'; el.textContent = text; return el; } function renderMessages(payload, append = false) { - const root = document.getElementById("messages-list"); - const sentinel = document.getElementById("scroll-sentinel"); + const root = document.getElementById('messages-list'); if (!root) return; + let sentinel = document.getElementById('scroll-sentinel'); + if (!sentinel) sentinel = makeScrollSentinel(); const channel = payload.channel; - document.getElementById("viewer-title").textContent = - channel?.name || payload.channel_id; - document.getElementById("viewer-subtitle").textContent = channel - ? `${channel.message_count} messages, latest: ${channel.last_date || "-"}` - : "No local database found for this channel yet."; + document.getElementById('viewer-title').textContent = channel?.name || payload.channel_id; + document.getElementById('viewer-subtitle').textContent = channel + ? `${payload.messages.length} shown${viewerState.search ? ` for "${viewerState.search}"` : ''}, ${channel.message_count} total, latest: ${channel.last_date || '-'}` + : 'No local database found for this channel yet.'; if (!append) { - root.innerHTML = ""; + root.innerHTML = ''; root.appendChild(sentinel); } let lastKey = null; if (append) { - const firstExisting = root.querySelector("[data-message-id]"); + const firstExisting = root.querySelector('[data-message-id]'); if (firstExisting) { - const candidate = firstExisting.closest(".chat-bubble-wrap"); + const candidate = firstExisting.closest('.chat-bubble-wrap'); if (candidate && candidate._dateKey) lastKey = candidate._dateKey; } } @@ -342,8 +390,16 @@ function renderMessages(payload, append = false) { if (payload.messages.length) { viewerState.oldestMessageId = payload.messages[0].message_id; - viewerState.newestMessageId = - payload.messages[payload.messages.length - 1].message_id; + viewerState.newestMessageId = payload.messages[payload.messages.length - 1].message_id; + } else if (!append) { + viewerState.oldestMessageId = null; + viewerState.newestMessageId = null; + const empty = document.createElement('div'); + empty.className = 'viewer-empty-state'; + empty.textContent = viewerState.search + ? `No messages found for "${viewerState.search}".` + : 'No saved messages in this channel yet.'; + root.appendChild(empty); } if (!append) { @@ -357,14 +413,11 @@ async function loadChannel(channelId, append = false) { viewerState.channelId = channelId; renderChannelList(); - const before = - append && viewerState.oldestMessageId - ? `&before=${viewerState.oldestMessageId}` - : ""; + const before = append && viewerState.oldestMessageId ? `&before=${viewerState.oldestMessageId}` : ''; try { const payload = await api(messageEndpoint(channelId, before)); if (append) { - const list = document.getElementById("messages-list"); + const list = document.getElementById('messages-list'); const prevScrollHeight = list.scrollHeight; renderMessages(payload, true); requestAnimationFrame(() => { @@ -374,14 +427,48 @@ async function loadChannel(channelId, append = false) { renderMessages(payload, false); } } catch (err) { - console.error("Failed to load messages:", err); + console.error('Failed to load messages:', err); } finally { viewerState.loading = false; } } +async function refreshCurrentChannel() { + if (!viewerState.channelId || viewerState.loading) return; + viewerState.oldestMessageId = null; + viewerState.newestMessageId = null; + await loadChannel(viewerState.channelId); +} + +function setupViewerTools() { + const searchInput = document.getElementById('viewer-search'); + const refreshBtn = document.getElementById('viewer-refresh-btn'); + const autoRefresh = document.getElementById('viewer-auto-refresh'); + let searchTimer = null; + + if (searchInput) { + searchInput.addEventListener('input', () => { + clearTimeout(searchTimer); + searchTimer = setTimeout(() => { + viewerState.search = searchInput.value.trim(); + refreshCurrentChannel(); + }, 250); + }); + } + + if (refreshBtn) { + refreshBtn.addEventListener('click', () => refreshCurrentChannel()); + } + + viewerState.autoRefreshTimer = window.setInterval(() => { + if (!autoRefresh || autoRefresh.checked) { + refreshCurrentChannel(); + } + }, 30000); +} + function setupInfiniteScroll() { - const sentinel = document.getElementById("scroll-sentinel"); + const sentinel = document.getElementById('scroll-sentinel'); if (!sentinel) return; if (viewerState.sentinelObserver) { viewerState.sentinelObserver.disconnect(); @@ -389,17 +476,12 @@ function setupInfiniteScroll() { viewerState.sentinelObserver = new IntersectionObserver( (entries) => { for (const entry of entries) { - if ( - entry.isIntersecting && - viewerState.channelId && - viewerState.oldestMessageId && - !viewerState.loading - ) { + if (entry.isIntersecting && viewerState.channelId && viewerState.oldestMessageId && !viewerState.loading) { loadChannel(viewerState.channelId, true); } } }, - { root: document.getElementById("messages-list"), threshold: 0.1 } + { root: document.getElementById('messages-list'), threshold: 0.1 }, ); viewerState.sentinelObserver.observe(sentinel); } @@ -408,27 +490,33 @@ async function main() { const params = new URLSearchParams(window.location.search); await loadViewerAccount(params); viewerState.channels = await api(channelsEndpoint()); - viewerState.channelId = - params.get("channel") || - viewerState.channels[0]?.channel_id || - null; + const requestedChannel = params.get('channel'); + const requestedChannelExists = + requestedChannel && viewerState.channels.some((channel) => channel.channel_id === requestedChannel); + viewerState.channelId = requestedChannelExists ? requestedChannel : viewerState.channels[0]?.channel_id || null; renderChannelList(); if (viewerState.channelId) { await loadChannel(viewerState.channelId); + } else { + resetMessageView( + 'No channels', + viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.', + ); } setupInfiniteScroll(); + setupViewerTools(); - const toggle = document.getElementById("sidebar-toggle"); - const sidebar = document.querySelector(".viewer-sidebar"); + const toggle = document.getElementById('sidebar-toggle'); + const sidebar = document.querySelector('.viewer-sidebar'); if (toggle && sidebar) { - toggle.addEventListener("click", () => { - sidebar.classList.toggle("open"); + toggle.addEventListener('click', () => { + sidebar.classList.toggle('open'); }); - sidebar.addEventListener("click", (e) => { + sidebar.addEventListener('click', (e) => { if (e.target === sidebar) { - sidebar.classList.remove("open"); + sidebar.classList.remove('open'); } }); } @@ -436,5 +524,5 @@ async function main() { main().catch((error) => { console.error(error); - alert(error.message); + showToast(error.message, 'error'); }); diff --git a/webui_server.py b/webui_server.py index 031e78d..f3b592d 100644 --- a/webui_server.py +++ b/webui_server.py @@ -194,6 +194,7 @@ def load_messages( channel_id: str, limit: int = 120, before_message_id: Optional[int] = None, + search: Optional[str] = None, ) -> List[Dict[str, Any]]: db_path = channel_db_path(account_id, channel_id) if not db_path.exists(): @@ -207,9 +208,15 @@ def load_messages( "FROM messages " ) params: List[Any] = [] + where: List[str] = [] if before_message_id is not None: - query += "WHERE message_id < ? " + where.append("message_id < ?") params.append(before_message_id) + if search: + where.append("message LIKE ?") + params.append(f"%{search}%") + if where: + query += "WHERE " + " AND ".join(where) + " " query += "ORDER BY message_id DESC LIMIT ?" params.append(limit) rows = conn.execute(query, params).fetchall() @@ -323,16 +330,29 @@ class JobRunner: def create_job(self, job_type: str, title: str, payload: Dict[str, Any]) -> Job: if self._shutdown_flag: raise RuntimeError("Server is shutting down, cannot create new jobs") - job_id = f"job-{int(time.time() * 1000)}" account_id = payload.get("account_id") - job = Job( - job_id=job_id, - job_type=job_type, - title=title, - payload=payload, - account_id=account_id, - ) with self.lock: + if account_id: + for job_id in self.job_order: + existing = self.jobs[job_id] + if ( + existing.account_id == account_id + and existing.status in {"queued", "running"} + ): + existing.logs = ( + existing.logs + + "\n" + + f"[{datetime.now().strftime('%H:%M:%S')}] Reused existing active job for this account." + ).strip() + return existing + job_id = f"job-{int(time.time() * 1000)}" + job = Job( + job_id=job_id, + job_type=job_type, + title=title, + payload=payload, + account_id=account_id, + ) self.jobs[job_id] = job self.job_order.insert(0, job_id) self.job_order = self.job_order[:20] @@ -346,6 +366,15 @@ class JobRunner: result = [j for j in result if j.get("account_id") == account_id] return result + def active_jobs_by_account(self) -> Dict[str, Dict[str, Any]]: + with self.lock: + active = {} + for job_id in self.job_order: + job = self.jobs[job_id] + if job.account_id and job.status in {"queued", "running"}: + active[job.account_id] = job.to_dict() + return active + def get_job(self, job_id: str) -> Optional[Dict[str, Any]]: with self.lock: job = self.jobs.get(job_id) @@ -731,6 +760,33 @@ class PerAccountContinuousScrapeManager: } store.update(mutate) + def refresh_config(self) -> None: + """ + Re-read account state from disk and sync the in-memory config. + Call this after external state changes (import, channel add/remove) + so the manager picks up new channels or config updates without a restart. + """ + disk_state = load_account(DATA_DIR, self.account_id) + disk_cfg = disk_state.get("continuous_scraping") or {} + with self.lock: + # Merge disk config into memory, preserving our current running state + self.config["enabled"] = bool(disk_cfg.get("enabled", False)) + self.config["interval_minutes"] = max(1, int(disk_cfg.get("interval_minutes", 1) or 1)) + self.config["channels"] = [ + str(item).strip() + for item in disk_cfg.get("channels", []) + if str(item).strip() + ] + self.config["run_all_tracked"] = bool(disk_cfg.get("run_all_tracked", True)) + # Sync running state with desired enabled state + if self.config["enabled"] and not self.status["running"]: + pass # don't auto-start — user must call start() + elif not self.config["enabled"] and self.status["running"]: + self._log("Continuous disabled via external state change, stopping.", "warn") + self.stop_event.set() + self.status["running"] = False + self._log("Config refreshed from disk.", "debug") + def _log(self, message: str, level: str = "debug") -> None: timestamp = utc_now_iso() entry = {"timestamp": timestamp, "level": level, "message": message} @@ -811,6 +867,23 @@ class PerAccountContinuousScrapeManager: def _run_loop(self) -> None: while not self.stop_event.is_set(): + # Refresh config from disk so channel / setting changes take effect + self.refresh_config() + + # Check auth — don't iterate if account isn't authorized + auth_info = auth_status_for(self.account_id) + if auth_info.get("status") not in ("ready", "authorized"): + self._log( + f"Account not authorized (status={auth_info.get('status')}), " + "skipping iteration", + "warn", + ) + sleep_seconds = max(5, 60) + interrupted = self.stop_event.wait(timeout=sleep_seconds) + if interrupted: + break + continue + channels = self._resolve_channels() cfg = self.snapshot()["config"] interval_minutes = cfg.get("interval_minutes", 1) @@ -893,6 +966,11 @@ class ContinuousScrapeOrchestrator: result[account_id] = dict(mgr.snapshot()) return result + def refresh_account(self, account_id: str) -> None: + """Re-read account state from disk and sync the continuous manager config.""" + mgr = self._get_or_create(account_id) + mgr.refresh_config() + def update_for( self, account_id: str, @@ -902,12 +980,14 @@ class ContinuousScrapeOrchestrator: run_all_tracked: bool, ) -> Dict[str, Any]: mgr = self._get_or_create(account_id) - return mgr.update(enabled, interval_minutes, channels, run_all_tracked) + result = mgr.update(enabled, interval_minutes, channels, run_all_tracked) + # After explicit update, sync state to disk is already done by mgr.update + return result - def add_account(self, account_id: str) -> None: + def add_account(self, account_id: str, auto_start: bool = True) -> None: acc_state = load_account(DATA_DIR, account_id) cfg = acc_state.get("continuous_scraping", {}) - if cfg.get("enabled", True) and START_CONTINUOUS: + if auto_start and cfg.get("enabled", False) and START_CONTINUOUS: self.start_account(account_id) def remove_account(self, account_id: str) -> None: @@ -924,7 +1004,7 @@ class ContinuousScrapeOrchestrator: for account_id in list_accounts(DATA_DIR): acc_state = load_account(DATA_DIR, account_id) cfg = acc_state.get("continuous_scraping", {}) - if cfg.get("enabled", True): + if cfg.get("enabled", False) and START_CONTINUOUS: self.start_account(account_id) @@ -973,6 +1053,27 @@ def auth_status_for(account_id: Optional[str] = None) -> Dict[str, Any]: return status +def account_health_summary(account_id: str, job_runner: Optional[JobRunner] = None) -> Dict[str, Any]: + acc_state = load_account(DATA_DIR, account_id) + acc_dir = account_data_dir(DATA_DIR, account_id) + session_file = Path(account_session_path(SESSION_DIR, account_id)) + channels = list_channels_snapshot(account_id) + active_job = (job_runner.active_jobs_by_account().get(account_id) if job_runner else None) + last_scrape = next((item.get("last_date") for item in channels if item.get("last_date")), None) + return { + "account_id": account_id, + "label": acc_state.get("label", ""), + "data_dir_exists": acc_dir.exists(), + "session_ready": session_file.exists(), + "api_credentials": bool(acc_state.get("api_id") and acc_state.get("api_hash")), + "channel_count": len(channels), + "message_count": sum(int(item.get("message_count") or 0) for item in channels), + "media_count": sum(int(item.get("media_count") or 0) for item in channels), + "last_scrape": last_scrape, + "active_job": active_job, + } + + def auth_status() -> Dict[str, Any]: return auth_status_for(account_id=None) @@ -1166,6 +1267,12 @@ def openapi_payload() -> Dict[str, Any]: "in": "query", "schema": {"type": "integer"}, }, + { + "name": "search", + "in": "query", + "schema": {"type": "string"}, + "description": "Full-text search in message text", + }, ], "responses": json_response, } @@ -1229,6 +1336,31 @@ def openapi_payload() -> Dict[str, Any]: }, } }, + "/api/jobs/{job_id}/events": { + "get": { + "summary": "Job SSE event stream", + "description": "Server-Sent Events stream of job status updates. Closes when job completes or fails.", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + } + ], + "responses": { + "200": { + "description": "SSE stream of job status objects", + "content": { + "text/event-stream": { + "schema": {"type": "string"} + } + }, + }, + "404": {"description": "Job not found"}, + }, + } + }, "/api/jobs/scrape": { "post": { "summary": "Scrape one channel or all tracked channels", @@ -1318,6 +1450,23 @@ def openapi_payload() -> Dict[str, Any]: "responses": {**json_response, **error_response}, }, }, + "/api/accounts/import": { + "post": { + "summary": "Import account from exported JSON", + "requestBody": json_body( + { + "account_id": {"type": "string", "example": "work"}, + "state": { + "type": "object", + "description": "Exported account state object", + "example": {"label": "Work", "api_id": 123456, "api_hash": "abc", "channels": {}, "continuous_scraping": {"enabled": False}}, + }, + }, + ["account_id", "state"], + ), + "responses": {**json_response, **error_response}, + } + }, "/api/accounts/{id}": { "get": { "summary": "Account dashboard", @@ -1344,6 +1493,34 @@ def openapi_payload() -> Dict[str, Any]: "responses": json_response, }, }, + "/api/accounts/{id}/export": { + "get": { + "summary": "Export account settings as JSON", + "parameters": [ + { + "name": "id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + } + ], + "responses": json_response, + } + }, + "/api/accounts/{id}/health": { + "get": { + "summary": "Account health summary", + "parameters": [ + { + "name": "id", + "in": "path", + "required": True, + "schema": {"type": "string"}, + } + ], + "responses": json_response, + } + }, "/api/accounts/{id}/auth": { "get": { "summary": "Account auth snapshot", @@ -1452,6 +1629,12 @@ def openapi_payload() -> Dict[str, Any]: "in": "query", "schema": {"type": "integer"}, }, + { + "name": "search", + "in": "query", + "schema": {"type": "string"}, + "description": "Full-text search in message text", + }, ], "responses": json_response, } @@ -1594,6 +1777,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if path == "/": return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8") + if path == "/settings": + return self.serve_file(WEBUI_DIR / "settings.html", "text/html; charset=utf-8") if path == "/viewer": return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8") if path in {"/swagger", "/swigger", "/docs"}: @@ -1634,6 +1819,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): ) if path == "/api/jobs": return self.send_json(self.app.job_runner.recent_jobs()) + if path.startswith("/api/jobs/") and path.endswith("/events"): + job_id = path.split("/")[-2] + return self.stream_job_events(job_id) if path.startswith("/api/jobs/"): job_id = path.rsplit("/", 1)[-1] job = self.app.job_runner.get_job(job_id) @@ -1644,14 +1832,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(list_channels_snapshot()) if path.startswith("/api/channels/") and path.endswith("/messages"): parts = path.split("/") - channel_id = parts[3] + channel_id = urllib.parse.unquote(parts[3]).lstrip("@") limit = max(1, min(int(query.get("limit", ["120"])[0]), 300)) before = query.get("before") before_message_id = int(before[0]) if before else None + search = (query.get("search") or query.get("q") or [""])[0].strip() payload = { "channel_id": channel_id, "messages": load_messages( - None, channel_id, limit=limit, before_message_id=before_message_id + None, channel_id, limit=limit, before_message_id=before_message_id, + search=search or None ), "channel": next( ( @@ -1674,7 +1864,6 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): def _legacy_auth_snapshot(self) -> Dict[str, Any]: state = load_state() - session_ready = (SESSION_DIR / "session.session").exists() return { "phase": "unknown", "status": "unknown", @@ -1703,6 +1892,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): "auth": auth_info, "status": auth_info.get("status", "unknown"), "continuous_running": cs_info.get("status", {}).get("running", False), + "health": account_health_summary(account_id, self.app.job_runner), }) return self.send_json({"accounts": accounts}) @@ -1719,10 +1909,19 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self._handle_get_account_dashboard(account_id) if sub == ["auth"]: return self.send_json(self.app.auth_manager.auth_state(account_id)) + if sub == ["health"]: + return self.send_json(account_health_summary(account_id, self.app.job_runner)) + if sub == ["export"]: + payload = { + "version": 1, + "account_id": account_id, + "state": load_account(DATA_DIR, account_id), + } + return self.send_json(payload) if sub == ["channels"]: return self._handle_get_account_channels(account_id) if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages": - channel_id = sub[1] + channel_id = urllib.parse.unquote(sub[1]).lstrip("@") return self._handle_get_account_channel_messages(account_id, channel_id, query) if sub == ["continuous"]: return self.send_json( @@ -1744,9 +1943,37 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): "jobs": self.app.job_runner.recent_jobs(account_id=account_id), "continuous": cs_info, "scrape_media": bool(acc_state.get("scrape_media", True)), + "health": account_health_summary(account_id, self.app.job_runner), } return self.send_json(payload) + def stream_job_events(self, job_id: str) -> None: + if not self.app.job_runner.get_job(job_id): + return self.send_error_json(HTTPStatus.NOT_FOUND, "Job not found") + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "keep-alive") + self.end_headers() + + last_payload = None + deadline = time.time() + 60 * 30 + while time.time() < deadline: + job = self.app.job_runner.get_job(job_id) + if not job: + break + payload = json.dumps(job, ensure_ascii=False) + if payload != last_payload: + try: + self.wfile.write(f"data: {payload}\n\n".encode("utf-8")) + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError, OSError): + break + last_payload = payload + if job.get("status") in {"completed", "failed"}: + break + time.sleep(1) + def _handle_get_account_channels(self, account_id: str) -> None: return self.send_json(list_channels_snapshot(account_id)) @@ -1756,10 +1983,12 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): limit = max(1, min(int(query.get("limit", ["120"])[0]), 300)) before = query.get("before") before_message_id = int(before[0]) if before else None + search = (query.get("search") or query.get("q") or [""])[0].strip() payload = { "channel_id": channel_id, "messages": load_messages( account_id, channel_id, limit=limit, before_message_id=before_message_id + , search=search or None ), "channel": next( ( @@ -1780,6 +2009,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if path == "/": return self.serve_file(WEBUI_DIR / "index.html", "text/html; charset=utf-8", head_only=True) + if path == "/settings": + return self.serve_file(WEBUI_DIR / "settings.html", "text/html; charset=utf-8", head_only=True) if path == "/viewer": return self.serve_file(WEBUI_DIR / "viewer.html", "text/html; charset=utf-8", head_only=True) if path in {"/swagger", "/swigger", "/docs"}: @@ -1997,6 +2228,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): # ── /api/accounts/* POST routes ───────────────────────────────── if path == "/api/accounts": return self._handle_post_accounts_create(body) + if path == "/api/accounts/import": + return self._handle_post_accounts_import(body) if path.startswith("/api/accounts/"): return self._handle_post_account(path, body) @@ -2034,14 +2267,53 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): "scrape_media": True, "forwarding_rules": [], "continuous_scraping": { - "enabled": True, + "enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True, }, } store.save(init_state) - self.app.continuous_orchestrator.add_account(account_id) + self.app.continuous_orchestrator.add_account(account_id, auto_start=False) + return self.send_json({"ok": True, "account_id": account_id}) + + def _handle_post_accounts_import(self, body: Dict[str, Any]) -> None: + account_id = str(body.get("account_id") or body.get("id") or "").strip() + state = body.get("state") + if not account_id: + return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id is required") + if not account_id.replace("-", "").replace("_", "").isalnum(): + return self.send_error_json(HTTPStatus.BAD_REQUEST, "account_id must be alphanumeric (dashes and underscores allowed)") + if account_exists(DATA_DIR, account_id): + return self.send_error_json(HTTPStatus.BAD_REQUEST, f"Account '{account_id}' already exists") + if not isinstance(state, dict): + return self.send_error_json(HTTPStatus.BAD_REQUEST, "state object is required") + + imported_state = { + "label": str(body.get("label") or state.get("label") or account_id).strip(), + "api_id": state.get("api_id"), + "api_hash": state.get("api_hash"), + "channels": state.get("channels") if isinstance(state.get("channels"), dict) else {}, + "channel_names": state.get("channel_names") if isinstance(state.get("channel_names"), dict) else {}, + "scrape_media": bool(state.get("scrape_media", True)), + "forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [], + "continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else { + "enabled": False, + "interval_minutes": 1, + "channels": [], + "run_all_tracked": True, + }, + } + + def mutate_global(global_state: Dict[str, Any]) -> None: + accounts = global_state.setdefault("accounts", []) + if account_id not in accounts: + accounts.append(account_id) + + get_global_store(DATA_DIR).update(mutate_global) + get_account_store(DATA_DIR, account_id).save(imported_state) + self.app.continuous_orchestrator.add_account(account_id, auto_start=False) + self.app.continuous_orchestrator.refresh_account(account_id) return self.send_json({"ok": True, "account_id": account_id}) def _handle_post_account(self, path: str, body: Dict[str, Any]) -> None: @@ -2141,6 +2413,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(payload) def _handle_account_channel_add(self, account_id: str, channel_id: str, name: Optional[str]) -> None: + channel_id = channel_id.lstrip("@") if not channel_id: return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") store = get_account_store(DATA_DIR, account_id) @@ -2150,6 +2423,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if name: state.setdefault("channel_names", {})[channel_id] = str(name).strip() store.update(mutate) + self.app.continuous_orchestrator.refresh_account(account_id) return self.send_json({"ok": True, "channel_id": channel_id}) def _handle_account_channel_remove(self, account_id: str, channel_id: str) -> None: @@ -2164,6 +2438,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): del chans[channel_id] state.get("channel_names", {}).pop(channel_id, None) store.update(mutate) + self.app.continuous_orchestrator.refresh_account(account_id) return self.send_json({"ok": existed[0], "channel_id": channel_id}) def _handle_account_job_scrape(self, account_id: str, body: Dict[str, Any]) -> None: @@ -2300,9 +2575,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if acc_dir.exists(): import shutil shutil.rmtree(str(acc_dir), ignore_errors=True) - if session_file.exists(): + for session_sidecar in ( + session_file, + Path(str(session_file) + "-wal"), + Path(str(session_file) + "-shm"), + session_file.with_suffix(session_file.suffix + "-journal"), + ): + if not session_sidecar.exists(): + continue try: - session_file.unlink() + session_sidecar.unlink() except OSError: pass