62 Commits

Author SHA1 Message Date
forust 70ac1ce73e feat(webui): add dashboard stat details
ci / lint-prettier (push) Successful in 11s
ci / lint-ruff (push) Successful in 5s
ci / lint-yaml (push) Successful in 6s
ci / lint-dockerfiles (push) Successful in 8s
ci / validate (push) Successful in 7s
ci / publish (push) Successful in 30s
2026-07-26 19:53:19 +02:00
forust 81e69865ce fix(webui): bust stylesheet cache
ci / lint-prettier (push) Successful in 26s
ci / lint-ruff (push) Successful in 11s
ci / lint-yaml (push) Successful in 18s
ci / lint-dockerfiles (push) Successful in 11s
ci / validate (push) Successful in 7s
ci / publish (push) Successful in 19s
2026-07-26 19:47:48 +02:00
forust 418c4d6192 fix(webui): prevent dashboard horizontal offset
ci / lint-prettier (push) Successful in 12s
ci / lint-ruff (push) Successful in 5s
ci / lint-yaml (push) Successful in 6s
ci / lint-dockerfiles (push) Successful in 6s
ci / validate (push) Successful in 11s
ci / publish (push) Successful in 16s
2026-07-26 19:45:23 +02:00
forust 5956ea36a8 style(webui): match dashboard reference
ci / lint-yaml (push) Successful in 6s
ci / lint-prettier (push) Successful in 11s
ci / lint-ruff (push) Successful in 5s
ci / lint-dockerfiles (push) Successful in 5s
ci / validate (push) Successful in 7s
ci / publish (push) Successful in 19s
2026-07-26 19:29:04 +02:00
forust 93c8d00af6 style(webui): format stylesheet
ci / lint-prettier (push) Successful in 11s
ci / lint-ruff (push) Successful in 5s
ci / lint-yaml (push) Successful in 9s
ci / lint-dockerfiles (push) Successful in 6s
ci / validate (push) Successful in 7s
ci / publish (push) Successful in 2m3s
2026-07-26 19:02:11 +02:00
forust 08f8196bf9 style(webui): match GPU Radar visual system
ci / lint-dockerfiles (push) Successful in 5s
ci / validate (push) Successful in 4s
ci / publish (push) Has been skipped
ci / lint-prettier (push) Failing after 11s
ci / lint-ruff (push) Successful in 4s
ci / lint-yaml (push) Successful in 5s
2026-07-26 18:56:41 +02:00
forust b963cf07df ci: restore docker-based lint and validate jobs
ci / lint-prettier (push) Successful in 6s
ci / lint-ruff (push) Successful in 3s
ci / lint-yaml (push) Successful in 3s
ci / lint-dockerfiles (push) Successful in 6s
ci / validate (push) Successful in 4s
ci / publish (push) Successful in 5s
2026-06-30 01:12:51 +02:00
forust 849bf97c97 ci: run lint and validate tools locally
ci / lint-dockerfiles (push) Failing after 1s
ci / validate (push) Failing after 1s
ci / publish (push) Has been skipped
ci / lint-prettier (push) Failing after 3s
ci / lint-ruff (push) Successful in 1s
ci / lint-yaml (push) Successful in 1s
2026-06-30 00:15:02 +02:00
forust 145e7de7f4 Harden scraper inputs and exports
ci / lint-prettier (push) Successful in 7s
ci / lint-ruff (push) Successful in 5s
ci / lint-yaml (push) Successful in 6s
ci / lint-dockerfiles (push) Successful in 4s
ci / validate (push) Successful in 4s
ci / publish (push) Successful in 8s
2026-06-29 17:27:43 +02:00
forust 27296caf69 Merge branch 'dev'
ci / lint-prettier (push) Successful in 7s
ci / lint-ruff (push) Successful in 4s
ci / lint-yaml (push) Successful in 5s
ci / validate (push) Successful in 4s
ci / lint-dockerfiles (push) Successful in 4s
ci / publish (push) Successful in 8s
2026-06-28 01:04:17 +02:00
forust a11ed6c5ba Simplify web UI theme
ci / lint-prettier (push) Successful in 8s
ci / lint-ruff (push) Successful in 5s
ci / lint-yaml (push) Successful in 4s
ci / lint-dockerfiles (push) Successful in 4s
ci / validate (push) Successful in 6s
ci / publish (push) Successful in 10s
2026-06-28 01:02:24 +02:00
forust 56f15e0ba5 Parallelize CI lint jobs
ci / lint-prettier (push) Successful in 8s
ci / lint-ruff (push) Successful in 4s
ci / lint-yaml (push) Successful in 5s
ci / lint-dockerfiles (push) Successful in 31s
ci / validate (push) Successful in 5s
ci / publish (push) Successful in 7s
2026-06-28 00:57:21 +02:00
forust 26b6344f7a Format web UI assets
ci / lint (push) Successful in 12s
ci / validate (push) Successful in 7s
ci / publish (push) Successful in 8s
2026-06-28 00:55:11 +02:00
forust 6bc8d6ed36 Polish web UI styling
ci / lint (push) Failing after 8s
ci / validate (push) Successful in 6s
ci / publish (push) Has been skipped
2026-06-28 00:53:54 +02:00
forust 01a0d9c6d4 Add CI workflow comment
ci / lint (push) Successful in 21s
ci / validate (push) Successful in 8s
ci / publish (push) Successful in 8s
2026-06-28 00:41:38 +02:00
forust d3beeb6b2d feat: show next continuous run on dashboard
ci / lint (push) Successful in 15s
ci / validate (push) Successful in 6s
ci / publish (push) Failing after 2s
2026-06-28 00:38:18 +02:00
forust 24d4898865 Revert "Run CI linters without Docker"
This reverts commit a74b8bfd14.
2026-06-28 00:37:48 +02:00
forust a74b8bfd14 Run CI linters without Docker 2026-06-28 00:36:17 +02:00
forust a2c5ca7f2b feat: add dedicated account settings page 2026-06-28 00:36:05 +02:00
forust 715cb448b7 Combine CI and publish workflows
ci / lint (push) Successful in 14s
ci / validate (push) Successful in 10s
ci / publish (push) Failing after 2s
2026-06-28 00:31:13 +02:00
forust ec4805f027 Add Docker publish pipeline 2026-06-27 23:01:57 +02:00
forust 5b27852b97 fix: URL-decode channel_id path params, strip @ prefix for state lookup 2026-06-27 21:53:52 +02:00
forust c5d0a7c7e5 docs: refresh API explorer for updated endpoints 2026-06-27 21:47:45 +02:00
forust 23bb370836 feat: polish multi-account dashboard and viewer 2026-06-27 21:47:45 +02:00
forust ff1ca13a03 feat: harden multi-account backend and add tests 2026-06-27 21:47:45 +02:00
forust 7aa5c84e63 fix: add search param to legacy /api/channels/{id}/messages endpoint 2026-06-27 21:28:08 +02:00
forust 881e7ddfca Harden account import/continuous behavior and finish multi-account UI polish 2026-06-27 21:26:18 +02:00
forust 59cdad220a Fix account tab persistence and viewer fallback 2026-06-27 19:09:07 +02:00
forust 9b72c3128e Merge branch 'dev' 2026-06-27 16:30:36 +02:00
forust 3eb6bdf534 docs: update README for multi-account, bump app.js cache version 2026-06-27 15:58:10 +02:00
forust f0d207bdea feat: add account selector dropdown in message viewer 2026-06-27 15:51:19 +02:00
forust 1bd385e054 fix: add WAL mode for session SQLite to prevent 'database is locked', persist active account tab in localStorage 2026-06-27 15:45:05 +02:00
forust 93f2b94dd2 chore: update entry points for multi-account
- main.py calls migrate_legacy_state() before starting server
- health.py accepts account_ids parameter for per-account health checks
- _continuous_check returns per-account status instead of single snapshot
2026-06-27 15:21:02 +02:00
forust eb69a89d91 feat: add multi-account message viewer support
- viewer.html shows account name in header
- viewer.js loads account list from /api/accounts, auto-selects first account
- Account-aware API endpoints (/api/accounts/{id}/... or legacy /api/...)
- Chat list redesign with avatar initials, preview text, time, message count
- loadViewerAccount() helper fetches account info and user ID
2026-06-27 15:20:56 +02:00
forust 62a6a492e0 feat: add per-account web UI with account tabs
- Account tab bar with tab switching (loadAccounts, switchAccount)
- Per-account dashboard panel showing auth status, channels, jobs, continuous
- Settings dialog with account management (add/delete accounts, credentials)
- Auth forms per account (QR, phone, password, set credentials)
- All action buttons with error handling (alert on failure)
- AMOLED dark theme with Telegram-style message bubbles
- Account tab and account list CSS styles
2026-06-27 15:20:49 +02:00
forust bb4e28a710 feat: add per-account API, auth, and continuous orchestration
- TelegramAuthManager rewritten for per-account clients/auth state
- ContinuousScrapeOrchestrator managing per-account continuous loops
- Full /api/accounts/* REST endpoints: CRUD, auth (QR/phone/password),
  channels, jobs, continuous config, settings, media toggle
- Legacy backward compat via legacy_account_id auto-detection
  (old /api/auth/*, /api/channels/*, /api/jobs/* still work for legacy
  single-account mode)
- load_messages() accepts account_id parameter for per-account data
- auth_status_for(), list_channels_snapshot() per-account helpers
- OpenAPI spec updated with all /api/accounts/* endpoints
2026-06-27 15:20:27 +02:00
forust 220d04b3a6 feat: make scraper and job service account-aware
- OptimizedTelegramScraper accepts account_id parameter;
  sets DATA_DIR to data/accounts/<id>/ when provided
- Session path uses account_session_path() for per-account session files
- initialize_client() reads per-account credentials from account state store
- ScraperJobService passes account_id from payload to scraper constructor
- Per-account state loaded from AccountStateStore before scraping
2026-06-27 15:20:15 +02:00
forust 6e3966aeb5 feat: add multi-account data layer with legacy migration
- GlobalStateStore for account registry in global state.json
- AccountStateStore for per-account settings (channels, credentials, continuous config)
- migrate_legacy_state() copies legacy data/ to data/accounts/default/
  with progress logging; renames session.session to default.session
- Helpers: account_data_dir(), account_session_path(), list_accounts(),
  load_account(), save_account(), get_account_store(), get_global_store()
- StateStore accepts defaults parameter for flexible default values
2026-06-27 15:20:09 +02:00
forust f5ef74bda6 Merge dev into main: lint fixes and config updates 2026-06-19 11:52:34 +02:00
forust ac5beb831b lint: fix bare except and unused variables 2026-06-19 11:52:30 +02:00
forust df644219de k8s + image 2026-06-11 18:19:17 +02:00
forust 7af597b800 sidebar toggle + css 2026-06-11 18:19:09 +02:00
forust c5361f2271 Redesign message viewer as Telegram-style chat with bubbles, replies, infinite scroll, and orientation
- Chat bubbles with 1px border-radius, angular tail arrows
- Own messages right (blue), others left (dark) via sender_id vs user_id
- Reply preview with left border accent, clickable scroll to original
- Infinite scroll up via IntersectionObserver on scroll-sentinel
- Date separators (Today/Yesterday/date) between messages
- Full-height messages panel with sidebar (channels) + header
- Backend: reply_to_message data in load_messages(), user_id exposed in auth snapshot
- Auto-scroll to bottom on channel load (triple scrollTop fallback)
- history.scrollRestoration = manual to prevent mid-page jumps
2026-05-27 01:01:18 +02:00
forust 1dfe41950e Add graceful shutdown, logging, streaming with Range Requests, and state.json cache
- Graceful shutdown: SIGTERM/SIGINT handlers stop continuous scraper,
  disconnect Telegram client, drain running jobs with timeout
- Logging: replace print() with logging module throughout web server,
  basicConfig in main.py with structured log format
- Streaming: chunked file reads (64KB) instead of loading entire files
  into memory; full HTTP Range Request support (206 Partial Content)
  for video seeking in the browser
- Cache: TTL-based in-memory cache (1s) for state.json reads in
  StateStore, thread-safe with existing RLock
- Bugfix: send_error_json now converts HTTPStatus to int for JSON
2026-05-27 00:14:26 +02:00
forust 856d04bd16 Fixed on-hover button style 2026-05-24 22:29:25 +02:00
forust d49fe28f08 Refactor server state and add health checks 2026-05-24 22:24:13 +02:00
forust a78a708757 Improve webui style and add api docs 2026-05-24 22:18:41 +02:00
forust 6851922a9e Merge branch 'dev' 2026-05-24 21:56:24 +02:00
forust ff000e88d4 Enchance weblog 2026-04-26 23:21:26 +02:00
forust cc99f752b3 Change exposed port to 7887 2026-04-26 21:45:12 +02:00
forust be02eed378 Add export button for channel data and update API integration
Co-authored-by: Copilot <copilot@github.com>
2026-04-25 22:12:14 +02:00
forust 5b8a95d73d Revert "Remove tty and stdin_open options"
This reverts commit 612fa28ec6.
2026-04-25 03:00:00 +02:00
forust cf674b4943 Update port for webui to 7887 2026-04-25 02:56:21 +02:00
forust 612fa28ec6 Remove tty and stdin_open options 2026-04-25 02:55:01 +02:00
forust 47ef106ee1 Add .dockerignore file to exclude unnecessary files and directories 2026-04-25 02:51:11 +02:00
forust 46922a887f Add tabbed navigation and update styles for content panels 2026-04-25 02:43:16 +02:00
forust 3565e31885 Remove funding config 2026-04-25 02:35:02 +02:00
forust 9094a52ccb Restore funding config 2026-04-25 02:34:45 +02:00
forust db3641ddf9 Add web auth and continuous scraping logs 2026-04-25 02:34:35 +02:00
forust 19e173212c Update README for web UI workflow 2026-04-25 02:19:58 +02:00
forust f57e1d63eb Add lightweight web control panel 2026-04-25 02:19:52 +02:00
forust 535b8195ef Prepare scraper for shared data and server mode 2026-04-25 02:19:43 +02:00
36 changed files with 11570 additions and 920 deletions
+215
View File
@@ -0,0 +1,215 @@
data/
.codex
.python-version
uv.lock
.uv-cache/
session/
session.session
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[codz]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py.cover
*.lcov
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
# Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
# poetry.lock
# poetry.toml
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
# pdm.lock
# pdm.toml
.pdm-python
.pdm-build/
# pixi
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
# pixi.lock
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
# in the .venv directory. It is recommended not to include this directory in version control.
.pixi/*
!.pixi/config.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule*
celerybeat.pid
# Redis
*.rdb
*.aof
*.pid
# RabbitMQ
mnesia/
rabbitmq/
rabbitmq-data/
# ActiveMQ
activemq-data/
# SageMath parsed files
*.sage.py
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
# .idea/
# Abstra
# Abstra is an AI-powered process automation framework.
# Ignore directories containing user credentials, local state, and settings.
# Learn more at https://abstra.io/docs
.abstra/
# Visual Studio Code
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
# and can be added to the global gitignore or merged into this file. However, if you prefer,
# you could uncomment the following to ignore the entire vscode folder
# .vscode/
# Temporary file for partial code execution
tempCodeRunnerFile.py
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
# Marimo
marimo/_static/
marimo/_lsp/
__marimo__/
# Streamlit
.streamlit/secrets.toml
+24
View File
@@ -0,0 +1,24 @@
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.{yml,yaml}]
indent_size = 2
[*.{json,jsonc}]
indent_size = 2
[*.md]
trim_trailing_whitespace = false
[*.py]
indent_size = 4
[{Makefile,makefile}]
indent_style = tab
+171
View File
@@ -0,0 +1,171 @@
name: ci
'on':
push:
branches:
- '**'
pull_request:
workflow_dispatch:
env:
# Shared image coordinates for the publish job.
REGISTRY: gcr.forust.xyz
IMAGE_NAME: forust/telegram-scraper
jobs:
lint-prettier:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check formatting with Prettier
shell: bash
run: |
mapfile -t prettier_files < <(
git ls-files \
| grep -E '\.(md|json|ya?ml|html|css)$' \
| grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
)
if [ "${#prettier_files[@]}" -eq 0 ]; then
echo "No Prettier-managed files found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
node:22-alpine \
sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
lint-ruff:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Python with Ruff
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/astral-sh/ruff:latest \
check .
lint-yaml:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint YAML syntax
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
cytopia/yamllint:latest \
-c .yamllint .
lint-dockerfiles:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Dockerfiles
shell: bash
run: |
mapfile -t dockerfiles < <(
git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
)
if [ "${#dockerfiles[@]}" -eq 0 ]; then
echo "No Dockerfiles found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
--entrypoint hadolint \
hadolint/hadolint:latest-debian \
-c .hadolint.yaml "${dockerfiles[@]}"
validate:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Validate Kubernetes manifests
shell: bash
run: |
mapfile -t manifests < <(
git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
)
if [ "${#manifests[@]}" -eq 0 ]; then
echo "No Kubernetes manifests found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/yannh/kubeconform:latest \
-strict \
-ignore-missing-schemas \
-summary \
"${manifests[@]}"
publish:
needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Read project version
id: version
shell: bash
run: |
version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')"
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Log in to registry
shell: bash
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-u "${{ secrets.REGISTRY_USERNAME }}" \
--password-stdin
- name: Build and push image
shell: bash
run: |
image="${REGISTRY}/${IMAGE_NAME}"
tags=("latest" "${{ steps.version.outputs.version }}")
case "${GITHUB_REF_NAME}" in
main)
tags+=("main" "prod")
;;
dev)
tags+=("dev")
;;
esac
build_args=()
for tag in "${tags[@]}"; do
build_args+=(-t "${image}:${tag}")
done
docker build "${build_args[@]}" .
for tag in "${tags[@]}"; do
docker push "${image}:${tag}"
done
-1
View File
@@ -1 +0,0 @@
ko_fi: unnohwn
+171
View File
@@ -0,0 +1,171 @@
name: ci
'on':
push:
branches:
- '**'
pull_request:
workflow_dispatch:
env:
# Shared image coordinates for the publish job.
REGISTRY: gcr.forust.xyz
IMAGE_NAME: forust/telegram-scraper
jobs:
lint-prettier:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check formatting with Prettier
shell: bash
run: |
mapfile -t prettier_files < <(
git ls-files \
| grep -E '\.(md|json|ya?ml|html|css)$' \
| grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
)
if [ "${#prettier_files[@]}" -eq 0 ]; then
echo "No Prettier-managed files found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
node:22-alpine \
sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
lint-ruff:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Python with Ruff
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/astral-sh/ruff:latest \
check .
lint-yaml:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint YAML syntax
shell: bash
run: |
docker run --rm \
-v "$PWD:/work" \
-w /work \
cytopia/yamllint:latest \
-c .yamllint .
lint-dockerfiles:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Lint Dockerfiles
shell: bash
run: |
mapfile -t dockerfiles < <(
git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
)
if [ "${#dockerfiles[@]}" -eq 0 ]; then
echo "No Dockerfiles found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
--entrypoint hadolint \
hadolint/hadolint:latest-debian \
-c .hadolint.yaml "${dockerfiles[@]}"
validate:
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Validate Kubernetes manifests
shell: bash
run: |
mapfile -t manifests < <(
git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
)
if [ "${#manifests[@]}" -eq 0 ]; then
echo "No Kubernetes manifests found."
exit 0
fi
docker run --rm \
-v "$PWD:/work" \
-w /work \
ghcr.io/yannh/kubeconform:latest \
-strict \
-ignore-missing-schemas \
-summary \
"${manifests[@]}"
publish:
needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
runs-on: [self-hosted, linux, arch, homelab]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Read project version
id: version
shell: bash
run: |
version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')"
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Log in to registry
shell: bash
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-u "${{ secrets.REGISTRY_USERNAME }}" \
--password-stdin
- name: Build and push image
shell: bash
run: |
image="${REGISTRY}/${IMAGE_NAME}"
tags=("latest" "${{ steps.version.outputs.version }}")
case "${GITHUB_REF_NAME}" in
main)
tags+=("main" "prod")
;;
dev)
tags+=("dev")
;;
esac
build_args=()
for tag in "${tags[@]}"; do
build_args+=(-t "${image}:${tag}")
done
docker build "${build_args[@]}" .
for tag in "${tags[@]}"; do
docker push "${image}:${tag}"
done
+228
View File
@@ -0,0 +1,228 @@
data/
.codex
.python-version
uv.lock
.uv-cache/
session/
session.session
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[codz]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py.cover
*.lcov
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
# Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
# poetry.lock
# poetry.toml
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
# pdm.lock
# pdm.toml
.pdm-python
.pdm-build/
# pixi
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
# pixi.lock
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
# in the .venv directory. It is recommended not to include this directory in version control.
.pixi/*
!.pixi/config.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule*
celerybeat.pid
# Redis
*.rdb
*.aof
*.pid
# RabbitMQ
mnesia/
rabbitmq/
rabbitmq-data/
# ActiveMQ
activemq-data/
# SageMath parsed files
*.sage.py
# Environments
.env
.envrc
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
# .idea/
# Abstra
# Abstra is an AI-powered process automation framework.
# Ignore directories containing user credentials, local state, and settings.
# Learn more at https://abstra.io/docs
.abstra/
# Visual Studio Code
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
# and can be added to the global gitignore or merged into this file. However, if you prefer,
# you could uncomment the following to ignore the entire vscode folder
# .vscode/
# Temporary file for partial code execution
tempCodeRunnerFile.py
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
# Marimo
marimo/_static/
marimo/_lsp/
__marimo__/
# Streamlit
.streamlit/secrets.toml
+10
View File
@@ -0,0 +1,10 @@
ignored:
- DL3008
- DL3042
- DL3018
- DL3059
trustedRegistries:
- docker.io
- ghcr.io
- quay.io
- gcr.forust.xyz
+8
View File
@@ -0,0 +1,8 @@
{
"default": true,
"MD013": false,
"MD024": false,
"MD033": false,
"MD041": false,
"MD046": false
}
+8
View File
@@ -0,0 +1,8 @@
bracketSameLine: true
htmlWhitespaceSensitivity: css
printWidth: 120
tabWidth: 2
singleQuote: true
trailingComma: all
proseWrap: preserve
endOfLine: lf
+34
View File
@@ -0,0 +1,34 @@
{
"[yaml]": {
"editor.tabSize": 2,
"editor.insertSpaces": true,
"editor.formatOnSave": true,
"editor.defaultFormatter": "redhat.vscode-yaml"
},
"[python]": {
"editor.tabSize": 4,
"editor.formatOnSave": true,
"editor.defaultFormatter": "charliermarsh.ruff"
},
"[dockerfile]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "exiasr.hadolint"
},
"[markdown]": {
"editor.wordWrap": "wordWrapColumn",
"editor.wordWrapColumn": 120
},
"yaml.schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"yaml.format.enable": true,
"yaml.validate": true,
"yaml.completion": true,
"prettier.bracketSameLine": true,
"prettier.htmlWhitespaceSensitivity": "css",
"prettier.printWidth": 120,
"editor.formatOnSave": true,
"files.autoSave": "onFocusChange",
"editor.wordWrapColumn": 120,
"editor.wordWrap": "wordWrapColumn"
}
+17
View File
@@ -0,0 +1,17 @@
extends: default
rules:
comments:
min-spaces-from-content: 1
comments-indentation: false
document-start: disable
line-length: disable
braces:
min-spaces-inside: 0
max-spaces-inside: 1
brackets:
min-spaces-inside: 0
max-spaces-inside: 1
indentation:
spaces: 2
indent-sequences: consistent
+39
View File
@@ -0,0 +1,39 @@
{
"tab_size": 2,
"soft_wrap": "prefer_line",
"preferred_line_length": 120,
"format_on_save": "on",
"languages": {
"YAML": {
"tab_size": 2,
"hard_tabs": false,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "yaml-language-server" }
}
},
"Python": {
"tab_size": 4,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "ruff" }
}
}
},
"lsp": {
"yaml-language-server": {
"settings": {
"yaml": {
"schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"validate": true,
"completion": true,
"format": {
"enable": true
}
}
}
}
}
}
+18
View File
@@ -0,0 +1,18 @@
FROM python:3.11-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8080
VOLUME ["/app/data"]
VOLUME ["/app/session"]
CMD ["python", "main.py"]
+186 -160
View File
@@ -1,207 +1,233 @@
# Telegram Channel Scraper 📱
# Telegram Scraper
A powerful Python script that allows you to scrape messages and media from Telegram channels using the Telethon library. Features include real-time continuous scraping, media downloading, and data export capabilities.
Telegram scraper on top of Telethon with:
```
___________________ _________
\__ ___/ _____/ / _____/
| | / \ ___ \_____ \
| | \ \_\ \/ \
|____| \______ /_______ /
\/ \/
- CLI workflow for scraping, exports, media recovery, and forwarding
- lightweight multi-account Web UI served by Python
- SQLite storage per tracked chat/channel
- Docker/Compose / Kubernetes setup for "run it and leave it on the server"
## What It Does
- Scrapes messages with metadata: views, forwards, reactions, post author
- Downloads media into per-channel folders
- Stores messages in SQLite
- Exports to CSV and JSON
- Supports continuous scraping
- Supports forwarding rules
- Multi-account support — multiple Telegram sessions side by side
- Lets you browse tracked channels and local exports in a web panel with account tabs
## Project Layout
```text
.
├── main.py # starts the web server
├── webui_server.py # HTTP server + REST API
├── webui/ # plain HTML/CSS/JS frontend
│ ├── index.html / app.js # main dashboard
│ ├── viewer.html / viewer.js # message viewer with account selector
│ └── swagger.html / swagger.js # API docs
├── telegram_scraper_with_forwarding.py
├── app_state.py # multi-account state management
├── scraper_jobs.py # async job runner
├── health.py # health check endpoints
├── data/
│ ├── state.json # global state (accounts list)
│ └── accounts/<id>/ # per-account data
│ ├── state.json # per-account settings & credentials
│ └── <channel_id>/ # SQLite DB + media
└── session/
├── <account_id>.session # per-account Telethon session
└── ... # (WAL journal mode for concurrency)
```
## What's New in v3.1 🎉
## Requirements
**Enhanced Message Data:**
- **Message statistics** - Captures views, forwards, and post_author for each message
- **Reactions support** - Records all emoji reactions with counts (e.g., "😀 12 👍 3")
- **Automatic database migration** - Seamlessly adds new columns to existing databases
- **Richer exports** - All new data included in CSV/JSON exports
**Improved Channel Management:**
- **Channel names displayed** - Shows channel names alongside IDs everywhere
- **Smart filtering** - List option now only shows Channels and Groups (no private chats)
- **channels_list.csv export** - Automatically saves channel list with names, IDs, usernames, and types
- **"all" selection** - Quickly add all listed channels at once
- **Better export naming** - Files now named as `ID_username.csv` and `ID_username.json`
**Bug Fixes:**
- **Fixed channel ID parsing** - Resolved "invalid literal for int()" error in fix missing media
- **Better entity resolution** - Handles both numeric IDs and channel usernames
- **Improved error messages** - Shows channel names with IDs for clearer debugging
## Features 🚀
- **QR Code & Phone Authentication** - Choose your preferred login method
- Scrape messages with full metadata (views, forwards, reactions, post author)
- Download media files with parallel processing and unique naming
- Real-time continuous scraping
- Export data to JSON and CSV formats with enhanced metadata
- SQLite database storage with automatic schema migration
- Resume capability (saves progress)
- Interactive menu with channel names and numbered selection
- Smart channel filtering (only shows channels/groups)
- Progress tracking with visual progress bars
- Automatic channels list export to CSV
## Prerequisites 📋
Before running the script, you'll need:
- Python 3.7 or higher
- Python 3.11+
- Telegram account
- API credentials from Telegram
- `api_id` and `api_hash` from https://my.telegram.org
### Required Python packages
## Telegram API Credentials
```
pip install -r requirements.txt
```
1. Go to https://my.telegram.org/auth
2. Sign in with your Telegram account
3. Open `API development tools`
4. Create an application
5. Save:
- `api_id`
- `api_hash`
## Getting Telegram API Credentials 🔑
## Install
1. Visit https://my.telegram.org/auth
2. Log in with your phone number
3. Click on "API development tools"
4. Fill in the form:
- App title: Your app name
- Short name: Your app short name
- Platform: Can be left as "Desktop"
- Description: Brief description of your app
5. Click "Create application"
6. You'll receive:
- `api_id`: A number
- `api_hash`: A string of letters and numbers
Keep these credentials safe, you'll need them to run the script!
### Local
## Setup and Running 🔧
1. Clone the repository:
```bash
git clone https://github.com/unnohwn/telegram-scraper.git
cd telegram-scraper
```
2. Install requirements:
```bash
pip install -r requirements.txt
```
3. Run the script:
Or with `uv`:
```bash
python telegram-scraper.py
uv sync
```
4. On first run, you'll be prompted to enter:
- Your API ID (from my.telegram.org)
- Your API Hash (from my.telegram.org)
- **Choose authentication method:**
- **QR Code** (Recommended) - Scan with your phone (no phone number needed)
- **Phone Number** - Traditional SMS verification
## Smoke Test
## Usage 📝
A smoke test is a quick "does it start and answer basic requests?" check. It does
not replace full tests, but it catches broken imports, routes, and JSON responses.
The script provides a clean interactive menu:
```
========================================
TELEGRAM SCRAPER
========================================
[S] Scrape channels
[C] Continuous scraping
[M] Media scraping: ON
[L] List & add channels
[R] Remove channels
[E] Export data
[T] Rescrape media
[Q] Quit
========================================
```bash
python scripts/smoke_test.py
```
### Channel Selection Made Easy 🔢
The smoke test starts the web server on `127.0.0.1:18080` and disables automatic
continuous scraping for that process.
Instead of typing long channel IDs, use numbers:
## Running
**Adding Channels:**
```
[1] Tech News (ID: -1002116176890, Type: Channel, Username: @technews)
[2] Python Dev (ID: -1001597139842, Type: Group, Username: @pythondev)
[3] Daily Updates (ID: -1002274713954, Type: Channel, Username: @dailyupdates)
### Web UI
Enter: 1,3 (adds channels 1 and 3)
Or: all (adds all listed channels)
Starts on `0.0.0.0:8080` by default:
```bash
python main.py
```
**Viewing Your Channels:**
```
[1] Tech News (ID: -1002116176890), Last Message ID: 5234, Messages: 12450
[2] Python Dev (ID: -1001597139842), Last Message ID: 8192, Messages: 45782
Or:
```bash
uv run python main.py
```
**Scraping Channels:**
- Single: `1`
- Multiple: `1,3,5`
- All: `all`
- Mix formats: `1,-1001597139842,3`
Open:
## Data Storage 💾
```text
http://<server-ip>:8080
```
### Database Structure
The current web panel includes:
Data is stored in SQLite databases, one per channel:
- Location: `./channelname/channelname.db`
- Optimized with indexes for fast queries
- WAL mode for better performance
- Schema includes: message_id, date, sender info, message text, media info, reply_to, post_author, views, forwards, reactions
- Automatic migration adds new columns to existing databases
- multi-account support with account tabs
- tracked channels overview per account
- background job queue
- scrape/export/media actions per account
- shared `scrape_media` toggle
- local message viewer with account selector
- API docs at `/swagger` and `/openapi.json`
- health checks at `/health` and `/health/continuous`
### Media Storage 📁
### CLI
Media files are stored with unique naming:
- Location: `./channelname/media/`
- Format: `{message_id}-{unique_id}-{original_name}.ext`
- **No more file overwrites** - Each file gets a unique name
If you still want the old interactive mode:
### Exported Data 📊
```bash
python telegram_scraper_with_forwarding.py
```
Export formats:
1. **CSV**: `./channelname/channelid_username.csv`
2. **JSON**: `./channelname/channelid_username.json`
3. **Channel List**: `./channels_list.csv` (automatically created when using [L] option)
## Docker / Kubernetes
All exports include complete message metadata: views, forwards, reactions, and post author information.
### Docker Compose
## Performance Features ⚙️
```bash
docker compose up -d --build
```
- **5 concurrent downloads** for faster media processing
- **Batch database operations** for optimal speed
- **Progress bars** with real-time feedback
- **Resume capability** - Continue where you left off
- **Memory-efficient** exports for large datasets
### Kubernetes
## Error Handling 🛠️
```bash
kubectl apply -f k8s/telegram-scraper.yaml
```
- Automatic retry with exponential backoff
- Rate limit compliance
- Network error recovery
- State preservation during interruptions
Then open:
## Limitations ⚠️
```text
http://<server-ip>:7887
```
- Respects Telegram's rate limits
- Can only access public channels or channels you're a member of
- Media download size limits apply as per Telegram's restrictions
### Volumes (Docker)
## License 📄
- `./data:/app/data` for databases, exports, and `state.json`
- `session:/app/session` for Telethon sessions
This project is licensed under the MIT License - see the LICENSE file for details.
If you already logged in before with the same compose volume and did not remove it, the session should be reused.
## Disclaimer ⚖️
## Multi-Account
This tool is for educational purposes only. Make sure to:
- Respect Telegram's Terms of Service
- Obtain necessary permissions before scraping
- Use responsibly and ethically
- Comply with data protection regulations
The app supports multiple Telegram accounts side by side. Each account has:
- its own `state.json` under `data/accounts/<id>/`
- its own Telethon session file `session/<id>.session`
- its own channel databases and media
Use the Web UI account tabs to switch between accounts, or create new ones in Settings.
Legacy single-account data is automatically migrated to the `default` account on first startup.
## Shared State
The Web UI and CLI share the same files:
- `data/state.json` (global — accounts list)
- `data/accounts/<id>/state.json` (per-account credentials and settings)
- `session/<id>.session` (per-account Telethon session, WAL journal mode)
That means:
- credentials persist in state files
- both entry points use the same tracked channels per account
- both entry points can reuse the same Telegram login sessions
## Data Storage
### SQLite
Each tracked channel per account gets its own database:
```text
data/accounts/<account_id>/<channel_id>/<channel_id>.db
```
Main fields include:
- `message_id`
- `date`
- `sender_id`
- `first_name`
- `last_name`
- `username`
- `message`
- `media_type`
- `media_path`
- `reply_to`
- `post_author`
- `views`
- `forwards`
- `reactions`
### Media
Media files are stored in:
```text
data/accounts/<account_id>/<channel_id>/media/
```
### Exports
Generated into the same channel folder:
- `data/accounts/<account_id>/<channel_id>/<channel_id>_<username>.csv`
- `data/accounts/<account_id>/<channel_id>/<channel_id>_<username>.json`
## Notes
- The web server is intentionally simple: plain HTML/CSS/JS, no frontend framework
- The viewer reads local SQLite data and supports account switching via a dropdown
- Session SQLite files use WAL journal mode to prevent "database is locked" errors when concurrent scraping and jobs run
- The active account tab is persisted in `localStorage` across page reloads
- If dependencies are installed but the Telegram session is missing, the Web UI falls back to read-only status until login is available
## Disclaimer
Use responsibly and make sure your usage complies with Telegram rules, local law, and any privacy obligations.
+313
View File
@@ -0,0 +1,313 @@
import json
import logging
import shutil
import threading
import time
from copy import deepcopy
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
logger = logging.getLogger(__name__)
# ── defaults ──────────────────────────────────────────────────────────
GLOBAL_DEFAULTS: Dict[str, Any] = {
"accounts": [],
"version": 2,
}
ACCOUNT_DEFAULTS: Dict[str, Any] = {
"label": "",
"api_id": None,
"api_hash": None,
"channels": {},
"channel_names": {},
"scrape_media": True,
"forwarding_rules": [],
"continuous_scraping": {
"enabled": False,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
},
}
# ── StateStore (thread-safe JSON) ─────────────────────────────────────
class StateStore:
"""Thread-safe JSON state store with TTL cache."""
def __init__(self, path: Path, defaults: Optional[Dict[str, Any]] = None):
self.path = path
self.defaults = defaults or {}
self.lock = threading.RLock()
self._cache: Optional[Dict[str, Any]] = None
self._cache_time: float = 0
self._cache_ttl: float = 1.0
def load(self) -> Dict[str, Any]:
with self.lock:
now = time.time()
if self._cache is not None and (now - self._cache_time) < self._cache_ttl:
return dict(self._cache)
if not self.path.exists():
result = deepcopy(self.defaults)
self._cache = result
self._cache_time = now
return result
try:
with self.path.open("r", encoding="utf-8") as handle:
state: Dict[str, Any] = json.load(handle)
except (json.JSONDecodeError, OSError):
result = deepcopy(self.defaults)
self._cache = result
self._cache_time = now
return result
result = self._merge_defaults(state)
self._cache = result
self._cache_time = now
return result
def save(self, state: Dict[str, Any]) -> None:
with self.lock:
self.path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
merged = self._merge_defaults(state)
with tmp_path.open("w", encoding="utf-8") as handle:
json.dump(merged, handle, ensure_ascii=False, indent=2)
handle.write("\n")
tmp_path.replace(self.path)
self._cache = None
def update(self, mutator: Callable[[Dict[str, Any]], None]) -> Dict[str, Any]:
with self.lock:
state = self.load()
mutator(state)
self.save(state)
return state
def continuous_config(self) -> Dict[str, Any]:
state = self.load()
return deepcopy(state.get("continuous_scraping") or ACCOUNT_DEFAULTS["continuous_scraping"])
def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
def mutate(state: Dict[str, Any]) -> None:
state["continuous_scraping"] = {
"enabled": bool(config.get("enabled", True)),
"interval_minutes": max(1, int(config.get("interval_minutes", 1) or 1)),
"channels": [
str(item).strip()
for item in config.get("channels", [])
if str(item).strip()
],
"run_all_tracked": bool(config.get("run_all_tracked", True)),
}
return self.update(mutate)["continuous_scraping"]
def _merge_defaults(self, state: Dict[str, Any]) -> Dict[str, Any]:
merged = deepcopy(self.defaults)
for key, value in state.items():
if isinstance(value, dict) and isinstance(merged.get(key), dict):
nested = deepcopy(merged[key])
nested.update(value)
merged[key] = nested
else:
merged[key] = value
return merged
# ── global state helpers ──────────────────────────────────────────────
_GLOBAL_STORE: Optional[StateStore] = None
def get_global_store(data_dir: Path) -> StateStore:
global _GLOBAL_STORE
if _GLOBAL_STORE is None:
_GLOBAL_STORE = StateStore(data_dir / "state.json", defaults=GLOBAL_DEFAULTS)
return _GLOBAL_STORE
def load_global(data_dir: Path) -> Dict[str, Any]:
return get_global_store(data_dir).load()
def save_global(data_dir: Path, state: Dict[str, Any]) -> None:
get_global_store(data_dir).save(state)
def list_accounts(data_dir: Path) -> List[str]:
return list(load_global(data_dir).get("accounts", []))
def account_exists(data_dir: Path, account_id: str) -> bool:
return account_id in list_accounts(data_dir)
# ── per-account state helpers ─────────────────────────────────────────
_ACCOUNT_STORES: Dict[str, StateStore] = {}
_account_stores_lock = threading.Lock()
def get_account_store(data_dir: Path, account_id: str) -> StateStore:
global _ACCOUNT_STORES
with _account_stores_lock:
key = f"{data_dir}:{account_id}"
if key not in _ACCOUNT_STORES:
store = StateStore(
data_dir / "accounts" / account_id / "state.json",
defaults=ACCOUNT_DEFAULTS,
)
_ACCOUNT_STORES[key] = store
return _ACCOUNT_STORES[key]
def load_account(data_dir: Path, account_id: str) -> Dict[str, Any]:
return get_account_store(data_dir, account_id).load()
def save_account(data_dir: Path, account_id: str, state: Dict[str, Any]) -> None:
get_account_store(data_dir, account_id).save(state)
def account_data_dir(data_dir: Path, account_id: str) -> Path:
"""Per-account directory for channel DBs, media, exports."""
return data_dir / "accounts" / account_id
def account_session_path(session_dir: Path, account_id: str) -> str:
"""Per-account Telethon session file. Always under session/<id>.session."""
return str(session_dir / f"{account_id}.session")
# ── MIGRATION (with data copy) ─────────────────────────────────────────
def _copy_channel_data(src_root: Path, dst_root: Path, channel_id: str) -> None:
"""Copy a single channel's DB + media from src_root to dst_root."""
src_ch = src_root / channel_id
dst_ch = dst_root / channel_id
if not src_ch.exists():
return
dst_ch.mkdir(parents=True, exist_ok=True)
# SQLite DB
db_name = f"{channel_id}.db"
src_db = src_ch / db_name
dst_db = dst_ch / db_name
if src_db.exists() and not dst_db.exists():
shutil.copy2(src_db, dst_db)
logger.info(" copied %s", db_name)
# Media directory
src_media = src_ch / "media"
dst_media = dst_ch / "media"
if src_media.exists() and src_media.is_dir():
dst_media.mkdir(parents=True, exist_ok=True)
for item in src_media.iterdir():
if item.is_file():
dst_file = dst_media / item.name
if not dst_file.exists():
shutil.copy2(item, dst_file)
# Exports (csv, json)
for ext in (".csv", ".json"):
for f in src_ch.glob(f"*{ext}"):
dst_f = dst_ch / f.name
if not dst_f.exists():
shutil.copy2(f, dst_f)
def migrate_legacy_state(data_dir: Path, session_dir: Path) -> bool:
"""
Run once at startup. Detects legacy data/state.json that has api_id
and migrates to multi-account format, COPYING channel DBs/media from
data/<channel_id>/ → data/accounts/default/<channel_id>/ .
Returns True if migration was performed.
"""
state_path = data_dir / "state.json"
if not state_path.exists():
return False
try:
with state_path.open("r", encoding="utf-8") as f:
raw: Dict[str, Any] = json.load(f)
except (json.JSONDecodeError, OSError):
return False
# Already migrated or fresh install with no legacy api_id
if raw.get("version") == 2 or not raw.get("api_id"):
return False
logger.info(
"=== Legacy state detected — migrating to multi-account format ==="
)
logger.info("This may take a while if you have many channels with media.")
# ── 1. Create per-account state for "default" ──────────────────────
acc_dir = data_dir / "accounts" / "default"
acc_dir.mkdir(parents=True, exist_ok=True)
acc_state = {
"label": "Default",
"api_id": raw.get("api_id"),
"api_hash": raw.get("api_hash"),
"channels": raw.get("channels", {}),
"channel_names": raw.get("channel_names", {}),
"scrape_media": raw.get("scrape_media", True),
"forwarding_rules": raw.get("forwarding_rules", []),
"continuous_scraping": raw.get(
"continuous_scraping",
{
"enabled": True,
"interval_minutes": 1,
"channels": [],
"run_all_tracked": True,
},
),
}
acc_state_path = acc_dir / "state.json"
with open(acc_state_path, "w", encoding="utf-8") as f:
json.dump(acc_state, f, ensure_ascii=False, indent=2)
f.write("\n")
logger.info(" ✓ data/accounts/default/state.json created")
# ── 2. Copy channel data ───────────────────────────────────────────
legacy_channels = list(raw.get("channels", {}).keys())
if legacy_channels:
logger.info(
" Copying %d channel(s) from data/<id>/ → data/accounts/default/<id>/ ...",
len(legacy_channels),
)
for i, channel_id in enumerate(legacy_channels, 1):
logger.info(" [%d/%d] channel %s", i, len(legacy_channels), channel_id)
_copy_channel_data(data_dir, acc_dir, channel_id)
logger.info(" ✓ Channel data copied")
else:
logger.info(" (no tracked channels to copy)")
# ── 3. Rewrite global state.json ───────────────────────────────────
global_state = {
"accounts": ["default"],
"version": 2,
}
with open(state_path, "w", encoding="utf-8") as f:
json.dump(global_state, f, ensure_ascii=False, indent=2)
f.write("\n")
logger.info(" ✓ data/state.json rewritten (accounts list only)")
# ── 4. Rename legacy session file ─────────────────────────────────
legacy_session = session_dir / "session.session"
if legacy_session.exists():
new_session = session_dir / "default.session"
if not new_session.exists():
legacy_session.rename(new_session)
logger.info(" ✓ session/session.session → session/default.session")
logger.info("=== Migration complete ===")
return True
+27
View File
@@ -0,0 +1,27 @@
services:
telegram-scraper:
build:
context: .
dockerfile: Dockerfile
container_name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
tty: true
user: '1000:1000'
restart: unless-stopped
ports:
- 7887:8080
healthcheck:
test:
[
CMD,
python,
-c,
"import http.client, json; c=http.client.HTTPConnection('localhost', 8080); c.request('GET', '/health'); r=c.getresponse(); exit(0) if json.loads(r.read()).get('ok') else exit(1)",
]
interval: 30s
timeout: 10s
retries: 3
start_period: 15s
volumes:
- ./data:/app/data
- ./session:/app/session
+112
View File
@@ -0,0 +1,112 @@
import sqlite3
from pathlib import Path
from typing import Any, Dict, List, Optional
from app_state import StateStore
def health_payload(
data_dir: Path,
session_dir: Path,
state_store: StateStore,
continuous_snapshot: Dict[str, Any],
job_queue_size: int,
account_ids: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""
Return a health-check payload for the application.
NOTE: This endpoint does NOT support message search or query parameters.
Health concerns itself with filesystem, database connectivity, job queue,
and account session state. Message search is handled by the message API
endpoints (/api/channels/*/messages, /api/accounts/*/channels/*/messages).
"""
checks = {
"data_dir": _dir_check(data_dir, writable=True),
"session_dir": _dir_check(session_dir, writable=True),
"state_file": _state_check(state_store),
"sqlite": _sqlite_check(),
"continuous": _continuous_check(continuous_snapshot),
"job_queue": {"ok": True, "size": job_queue_size},
}
# Per-account health
if account_ids:
account_checks: Dict[str, Any] = {}
for acc_id in account_ids:
acc_dir = data_dir / "accounts" / acc_id
session_file = session_dir / f"{acc_id}.session"
account_checks[acc_id] = {
"data_dir": _dir_check(acc_dir),
"session_file": {
"ok": session_file.exists(),
"path": str(session_file),
},
}
checks["accounts"] = {
"ok": all(
item.get("data_dir", {}).get("ok", False)
for item in account_checks.values()
),
"items": account_checks,
}
ok = all(item.get("ok", False) for item in checks.values())
return {"ok": ok, "status": "ok" if ok else "degraded", "checks": checks}
def _dir_check(path: Path, writable: bool = False) -> Dict[str, Any]:
path.mkdir(parents=True, exist_ok=True)
ok = path.exists() and path.is_dir()
payload: Dict[str, Any] = {"ok": ok, "path": str(path)}
if writable:
probe = path / ".healthcheck"
try:
probe.write_text("ok", encoding="utf-8")
probe.unlink(missing_ok=True)
payload["writable"] = True
except OSError as exc:
payload.update({"ok": False, "writable": False, "error": str(exc)})
return payload
def _state_check(state_store: StateStore) -> Dict[str, Any]:
try:
state = state_store.load()
return {
"ok": True,
"path": str(state_store.path),
"has_api_credentials": bool(state.get("api_id") and state.get("api_hash")),
"tracked_channels": len(state.get("channels", {})),
}
except Exception as exc:
return {"ok": False, "path": str(state_store.path), "error": str(exc)}
def _sqlite_check() -> Dict[str, Any]:
try:
conn = sqlite3.connect(":memory:")
conn.execute("SELECT 1")
conn.close()
return {"ok": True}
except sqlite3.Error as exc:
return {"ok": False, "error": str(exc)}
def _continuous_check(snapshot: Dict[str, Any]) -> Dict[str, Any]:
running_accounts = snapshot.get("running_accounts")
if running_accounts is None:
running_accounts = snapshot
return {
"ok": True,
"account_count": len(running_accounts),
"accounts": {
aid: {
"running": bool((info.get("status") or info).get("running")),
"enabled": bool((info.get("config") or {}).get("enabled", False)),
"last_error": (info.get("status") or info).get("last_error"),
}
for aid, info in running_accounts.items()
if isinstance(info, dict)
},
}
+89
View File
@@ -0,0 +1,89 @@
apiVersion: v1
kind: Namespace
metadata:
name: telegram-scraper
---
apiVersion: v1
kind: Service
metadata:
name: telegram-scraper-service
namespace: telegram-scraper
spec:
selector:
app: telegram-scraper
ports:
- port: 8080
targetPort: 8080
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: telegram-scraper-statefulset
namespace: telegram-scraper
spec:
selector:
matchLabels:
app: telegram-scraper
serviceName: telegram-scraper-service
replicas: 1
template:
metadata:
labels:
app: telegram-scraper
spec:
containers:
- name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
stdin: true
tty: true
ports:
- containerPort: 8080
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 15
periodSeconds: 30
readinessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: telegram-scraper-data
mountPath: /app/data
- name: telegram-scraper-session
mountPath: /app/session
volumeClaimTemplates:
- metadata:
name: telegram-scraper-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
- metadata:
name: telegram-scraper-session
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Mi
---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: telegram-scraper-route
namespace: telegram-scraper
spec:
entryPoints:
- websecure
routes:
- match: Host(`tg.workstation.internal`)
kind: Rule
services:
- name: telegram-scraper-service
port: 8080
+33
View File
@@ -0,0 +1,33 @@
import logging
import sys
from pathlib import Path
from webui_server import run_server
def main():
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
stream=sys.stdout,
)
logger = logging.getLogger(__name__)
# Run legacy migration before starting the server
from app_state import migrate_legacy_state
data_dir = Path("data")
session_dir = Path("session")
try:
if migrate_legacy_state(data_dir, session_dir):
logger.info("Legacy migration completed successfully.")
except Exception as exc:
logger.error("Migration failed: %s", exc)
run_server()
if __name__ == "__main__":
main()
+25
View File
@@ -0,0 +1,25 @@
[project]
name = "telegram-scraper"
version = "2.0.0"
description = "Telegram-scraper with webui"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"aiohappyeyeballs==2.6.1",
"aiohttp==3.12.14",
"aiosignal==1.4.0",
"attrs==25.3.0",
"frozenlist==1.7.0",
"idna==3.10",
"multidict==6.6.3",
"propcache==0.3.2",
"pyaes==1.6.1",
"pyasn1==0.6.1",
"qrcode==8.0",
"rsa==4.9.1",
"Telethon==1.40.0",
"yarl==1.20.1",
]
[tool.pytest.ini_options]
pythonpath = ["."]
+1 -2
View File
@@ -1,8 +1,7 @@
aiohappyeyeballs==2.6.1
aiohttp==3.12.14
aiosignal==1.4.0
asyncio==3.4.3
attrs==25.3.0
attrs==25.3.0
frozenlist==1.7.0
idna==3.10
multidict==6.6.3
+83
View File
@@ -0,0 +1,83 @@
import asyncio
import logging
from typing import Any, Dict, List, Optional
from app_state import StateStore
logger = logging.getLogger(__name__)
class ScraperJobService:
def __init__(self, state_store: StateStore):
self.state_store = state_store
def run(self, job_type: str, payload: Dict[str, Any]) -> None:
if job_type == "set_scrape_media":
value = bool(payload["value"])
def mutate(state: Dict[str, Any]) -> None:
state["scrape_media"] = value
self.state_store.update(mutate)
logger.info("Media scraping set to %s", value)
return
asyncio.run(self._run_async(job_type, payload))
async def _run_async(self, job_type: str, payload: Dict[str, Any]) -> None:
# Extract account_id from payload, default to None (legacy)
account_id: Optional[str] = payload.get("account_id")
ScraperClass = self._import_scraper_class()
scraper = ScraperClass(account_id=account_id)
if account_id:
from app_state import load_account
acc_state = load_account(self.state_store.path.parent, account_id)
# Load per-account state into scraper state
scraper.state = acc_state
initialized = await scraper.initialize_client(interactive=False)
if not initialized:
raise RuntimeError(
"Telegram client is not ready. Check credentials, session, and write access."
)
try:
if job_type == "scrape_channel":
await self._scrape_channels(scraper, [payload["channel_id"]])
elif job_type == "scrape_all":
await self._scrape_channels(
scraper, list(scraper.state.get("channels", {}).keys())
)
elif job_type == "scrape_selected":
await self._scrape_channels(
scraper,
[str(channel_id) for channel_id in payload.get("channels", [])],
)
elif job_type == "export_all":
await scraper.export_data()
elif job_type == "export_channel":
channel_id = payload["channel_id"]
scraper.export_to_csv(channel_id)
scraper.export_to_json(channel_id)
elif job_type == "rescrape_media":
await scraper.rescrape_media(payload["channel_id"])
elif job_type == "fix_missing_media":
await scraper.fix_missing_media(payload["channel_id"])
elif job_type == "refresh_dialogs":
await scraper.list_channels()
else:
raise RuntimeError(f"Unsupported job type: {job_type}")
finally:
scraper.close_db_connections()
if scraper.client:
await scraper.client.disconnect()
async def _scrape_channels(self, scraper, channels: List[str]) -> None:
for channel_id in channels:
offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0)
await scraper.scrape_channel(channel_id, offset)
def _import_scraper_class(self):
from telegram_scraper_with_forwarding import OptimizedTelegramScraper
return OptimizedTelegramScraper
+85
View File
@@ -0,0 +1,85 @@
import json
import os
import subprocess
import sys
import time
import urllib.request
PORT = os.environ.get("TELEGRAM_SCRAPER_SMOKE_PORT", "18080")
BASE_URL = f"http://127.0.0.1:{PORT}"
ENDPOINTS = [
"/",
"/viewer",
"/swagger",
"/openapi.json",
"/health",
"/health/continuous",
"/api/dashboard",
"/api/continuous",
"/api/channels",
]
def fetch(path: str) -> tuple[int, bytes]:
with urllib.request.urlopen(BASE_URL + path, timeout=3) as response:
return response.status, response.read()
def wait_for_server(process: subprocess.Popen) -> None:
deadline = time.time() + 15
last_error = None
while time.time() < deadline:
if process.poll() is not None:
raise RuntimeError(f"server exited with code {process.returncode}")
try:
fetch("/health")
return
except Exception as exc:
last_error = exc
time.sleep(0.3)
raise RuntimeError(f"server did not become ready: {last_error}")
def main() -> int:
env = {
**os.environ,
"TELEGRAM_SCRAPER_PORT": PORT,
"TELEGRAM_SCRAPER_START_CONTINUOUS": "0",
}
process = subprocess.Popen(
[sys.executable, "main.py"],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
try:
wait_for_server(process)
for endpoint in ENDPOINTS:
status, body = fetch(endpoint)
if status != 200:
raise RuntimeError(f"{endpoint} returned HTTP {status}")
if (
endpoint.endswith(".json")
or endpoint.startswith("/api")
or endpoint.startswith("/health")
):
json.loads(body.decode("utf-8"))
print(f"ok {endpoint}")
return 0
except Exception as exc:
print(f"smoke test failed: {exc}", file=sys.stderr)
if process.stdout:
print(process.stdout.read(), file=sys.stderr)
return 1
finally:
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
if __name__ == "__main__":
raise SystemExit(main())
+426 -250
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+638
View File
@@ -0,0 +1,638 @@
"""
Smoke / integration tests for the multi-account Telegram scraper core logic.
These tests do NOT require Telegram credentials or network access.
They verify:
- Account creation and state isolation
- Account import safety (no auto-start continuous)
- Account export round-trip
- Account deletion (sidecar file cleanup)
- Active account persistence helpers
- Message search SQL construction
- Health payload structure
- Continuous orchestrator worker deduplication
- load_messages() pagination with search filter
"""
import os
import shutil
import sqlite3
import sys
import tempfile
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
from unittest.mock import MagicMock
# ── Mock heavy dependencies before any webui_server import ─────────────
# We need to mock qrcode and telethon because they aren't installed in CI.
sys.modules["qrcode"] = MagicMock()
sys.modules["qrcode.image"] = MagicMock()
sys.modules["qrcode.image.svg"] = MagicMock()
sys.modules["telethon"] = MagicMock()
sys.modules["telethon.errors"] = MagicMock()
sys.modules["telethon.errors"].SessionPasswordNeededError = type("SessionPasswordNeededError", (Exception,), {})
sys.modules["telegram_scraper_with_forwarding"] = MagicMock()
sys.modules["telegram_scraper_with_forwarding"]._ensure_session_wal = lambda p: None
sys.modules["telegram_scraper_with_forwarding"].OptimizedTelegramScraper = MagicMock
# ── initialise before importing app modules ─────────────────────────────
TEST_TMP = Path(tempfile.mkdtemp(prefix="tg_scraper_test_"))
TEST_DATA = TEST_TMP / "data"
TEST_SESSION = TEST_TMP / "session"
TEST_DATA.mkdir(parents=True, exist_ok=True)
TEST_SESSION.mkdir(parents=True, exist_ok=True)
os.environ["TELEGRAM_SCRAPER_HOST"] = "127.0.0.1"
os.environ["TELEGRAM_SCRAPER_PORT"] = "0"
os.environ["TELEGRAM_SCRAPER_START_CONTINUOUS"] = "0"
# Now safe to import app modules
import app_state # noqa: E402
from app_state import ( # noqa: E402
ACCOUNT_DEFAULTS,
account_data_dir,
account_exists,
account_session_path,
get_account_store,
get_global_store,
list_accounts,
load_account,
)
from health import health_payload # noqa: E402
def _init_test_env():
"""Reset global caches and re-initialise for a clean test."""
app_state._GLOBAL_STORE = None
with app_state._account_stores_lock:
app_state._ACCOUNT_STORES.clear()
_init_test_env()
# ── Helpers ────────────────────────────────────────────────────────────────
def make_account_id() -> str:
return f"test-{int(time.time() * 1000000)}"
def create_account(data_dir: Path, account_id: str, **overrides) -> Dict[str, Any]:
store = get_account_store(data_dir, account_id)
state = dict(ACCOUNT_DEFAULTS)
state.update(overrides)
state["label"] = overrides.get("label", account_id)
store.save(state)
def mutate(g: Dict[str, Any]) -> None:
accounts = g.setdefault("accounts", [])
if account_id not in accounts:
accounts.append(account_id)
get_global_store(data_dir).update(mutate)
return load_account(data_dir, account_id)
def create_channel_db(data_dir: Path, account_id: Optional[str], channel_id: str, messages: List[Dict[str, Any]]) -> None:
if account_id:
db_dir = account_data_dir(data_dir, account_id) / channel_id
else:
db_dir = data_dir / channel_id
db_dir.mkdir(parents=True, exist_ok=True)
db_path = db_dir / f"{channel_id}.db"
conn = sqlite3.connect(str(db_path))
conn.execute(
"""CREATE TABLE IF NOT EXISTS messages (
message_id INTEGER PRIMARY KEY,
date TEXT, sender_id INTEGER,
first_name TEXT, last_name TEXT, username TEXT,
message TEXT, media_type TEXT, media_path TEXT,
reply_to INTEGER, post_author TEXT,
views INTEGER, forwards INTEGER, reactions TEXT
)"""
)
for m in messages:
conn.execute(
"INSERT OR IGNORE INTO messages (message_id, date, message) VALUES (?, ?, ?)",
(m["message_id"], m.get("date"), m.get("message")),
)
conn.commit()
conn.close()
# ── Fixture setup / teardown ────────────────────────────────────────────────
def setup_function():
_init_test_env()
for child in TEST_DATA.iterdir():
if child.is_dir():
shutil.rmtree(child, ignore_errors=True)
else:
child.unlink(missing_ok=True)
def teardown_function():
_init_test_env()
# ═══════════════════════════════════════════════════════════════════════════
# Tests
# ═══════════════════════════════════════════════════════════════════════════
class TestAccountLifecycle:
"""Account creation, isolation, import, export, deletion."""
def test_create_account_minimal(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
assert account_exists(TEST_DATA, aid)
assert aid in list_accounts(TEST_DATA)
state = load_account(TEST_DATA, aid)
assert state["label"] == aid
assert state["continuous_scraping"]["enabled"] is False
def test_account_state_isolation(self):
a1, a2 = make_account_id(), make_account_id()
create_account(TEST_DATA, a1, label="Alpha", api_id=111)
create_account(TEST_DATA, a2, label="Beta", api_id=222)
s1 = load_account(TEST_DATA, a1)
s2 = load_account(TEST_DATA, a2)
assert s1["label"] == "Alpha"
assert s1["api_id"] == 111
assert s2["label"] == "Beta"
assert s2["api_id"] == 222
# channels are segregated — mutate one, verify other unchanged
s1.setdefault("channels", {})["-100aaa"] = 1
get_account_store(TEST_DATA, a1).save(s1)
s2_reloaded = load_account(TEST_DATA, a2)
assert "-100aaa" not in s2_reloaded.get("channels", {})
def test_delete_account_cleans_session_sidecars(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
session_path = Path(account_session_path(TEST_SESSION, aid))
session_path.write_text("session-data")
Path(str(session_path) + "-wal").write_text("wal")
Path(str(session_path) + "-shm").write_text("shm")
Path(str(session_path) + "-journal").write_text("journal")
acc_dir = account_data_dir(TEST_DATA, aid)
if acc_dir.exists():
shutil.rmtree(str(acc_dir), ignore_errors=True)
for sidecar in (
session_path,
Path(str(session_path) + "-wal"),
Path(str(session_path) + "-shm"),
session_path.with_suffix(session_path.suffix + "-journal"),
):
if sidecar.exists():
sidecar.unlink()
assert not session_path.exists()
assert not Path(str(session_path) + "-wal").exists()
assert not Path(str(session_path) + "-shm").exists()
assert not Path(str(session_path) + "-journal").exists()
assert not acc_dir.exists()
def test_delete_account_skips_missing_sidecars(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
session_path = Path(account_session_path(TEST_SESSION, aid))
session_path.write_text("session-only")
acc_dir = account_data_dir(TEST_DATA, aid)
if acc_dir.exists():
shutil.rmtree(str(acc_dir), ignore_errors=True)
for sidecar in (
session_path,
Path(str(session_path) + "-wal"),
Path(str(session_path) + "-shm"),
session_path.with_suffix(session_path.suffix + "-journal"),
):
if sidecar.exists():
sidecar.unlink()
assert not session_path.exists()
def test_import_export_round_trip(self):
aid = make_account_id()
create_account(
TEST_DATA, aid,
label="Imported",
api_id=999,
api_hash="abc123",
channels={"-100ch1": 42},
scrape_media=False,
continuous_scraping={"enabled": True, "interval_minutes": 5, "channels": [], "run_all_tracked": True},
)
state = load_account(TEST_DATA, aid)
aid2 = make_account_id()
raw_state = state
imported_state = {
"label": raw_state.get("label", aid2),
"api_id": raw_state.get("api_id"),
"api_hash": raw_state.get("api_hash"),
"channels": raw_state.get("channels", {}),
"channel_names": raw_state.get("channel_names", {}),
"scrape_media": bool(raw_state.get("scrape_media", True)),
"forwarding_rules": raw_state.get("forwarding_rules", []),
"continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else {
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
},
}
get_account_store(TEST_DATA, aid2).save(imported_state)
def mutate_global(g):
g.setdefault("accounts", []).append(aid2)
get_global_store(TEST_DATA).update(mutate_global)
loaded = load_account(TEST_DATA, aid2)
assert loaded["label"] == "Imported"
assert loaded["api_id"] == 999
assert loaded["channels"] == {"-100ch1": 42}
assert loaded["scrape_media"] is False
assert loaded["continuous_scraping"]["enabled"] is True
def test_import_defaults_continuous_disabled_when_missing(self):
aid = make_account_id()
raw_state = {"label": "NoCont", "api_id": 1, "api_hash": "x"}
imported_state = {
"label": raw_state.get("label", aid),
"api_id": raw_state.get("api_id"),
"api_hash": raw_state.get("api_hash"),
"channels": {},
"channel_names": {},
"scrape_media": True,
"forwarding_rules": [],
"continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else {
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
},
}
get_account_store(TEST_DATA, aid).save(imported_state)
def mutate_global(g):
g.setdefault("accounts", []).append(aid)
get_global_store(TEST_DATA).update(mutate_global)
loaded = load_account(TEST_DATA, aid)
assert loaded["continuous_scraping"]["enabled"] is False
assert loaded["continuous_scraping"]["run_all_tracked"] is True
class TestInputSafety:
"""Small safety checks for values that flow into files or exports."""
def test_channel_id_rejects_path_characters(self):
import webui_server as ws_module
assert ws_module.normalize_channel_id("@valid_name") == "valid_name"
for bad in ("../escape", "nested/channel", r"nested\channel", ".", "..", ""):
try:
ws_module.normalize_channel_id(bad)
except ValueError:
continue
raise AssertionError(f"accepted unsafe channel_id: {bad!r}")
def test_export_account_state_redacts_api_hash_by_default(self):
import webui_server as ws_module
state = {"label": "Work", "api_id": 123, "api_hash": "secret", "channels": {}}
redacted = ws_module.export_account_state(state)
assert redacted["api_hash"] is None
assert redacted["api_hash_present"] is True
assert state["api_hash"] == "secret"
full = ws_module.export_account_state(state, include_secrets=True)
assert full["api_hash"] == "secret"
assert "api_hash_present" not in full
class TestContinuousOrchestrator:
"""Continuous scraping manager safety using mocked webui_server."""
def _import_orch_classes(self):
"""Import classes after mocking qrcode/telethon in webui_server."""
import webui_server as ws_module
return ws_module.PerAccountContinuousScrapeManager, ws_module.ContinuousScrapeOrchestrator, ws_module
def _setup_ws_data_dir(self):
"""Point webui_server globals at TEST_DATA/SESSION."""
import webui_server as ws_module
self._ws_orig_data = ws_module.DATA_DIR
self._ws_orig_session = ws_module.SESSION_DIR
ws_module.DATA_DIR = TEST_DATA
ws_module.SESSION_DIR = TEST_SESSION
ws_module.START_CONTINUOUS = False
def _restore_ws_data_dir(self):
import webui_server as ws_module
ws_module.DATA_DIR = self._ws_orig_data
ws_module.SESSION_DIR = self._ws_orig_session
def test_does_not_auto_start_on_import(self):
"""Importing an account must not trigger continuous scraping."""
PerAccountContinuousScrapeManager, ContinuousScrapeOrchestrator, ws = self._import_orch_classes()
self._setup_ws_data_dir()
try:
orch = ContinuousScrapeOrchestrator()
aid = make_account_id()
create_account(TEST_DATA, aid, continuous_scraping={
"enabled": True, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
})
orch.add_account(aid, auto_start=False)
snap = orch.snapshot_for(aid)
assert snap["status"]["running"] is False
finally:
self._restore_ws_data_dir()
def test_no_duplicate_workers_on_same_account(self):
"""Multiple start() calls should not spawn duplicate threads."""
PerAccountContinuousScrapeManager, _, ws = self._import_orch_classes()
self._setup_ws_data_dir()
try:
aid = make_account_id()
# Must set enabled=True or refresh_config() will stop the thread
create_account(TEST_DATA, aid, continuous_scraping={
"enabled": True, "interval_minutes": 60, "channels": [], "run_all_tracked": True,
})
mgr = PerAccountContinuousScrapeManager(aid)
# Override refresh_config to prevent auth check from interfering
original_refresh = mgr.refresh_config
mgr.refresh_config = lambda: None
mgr.start()
t1 = mgr.thread
mgr.start() # second call — should be no-op
t2 = mgr.thread
assert t1 is t2, "start() spawned a second thread"
# Also verify only one thread is alive for this manager
alive_count = sum(
1 for t in threading.enumerate()
if t is t1 or t is t2
)
assert alive_count <= 1, "duplicate threads detected"
mgr.stop()
mgr.refresh_config = original_refresh
finally:
self._restore_ws_data_dir()
def test_disabled_account_not_started(self):
"""start_all() must not start accounts with enabled=False."""
_, ContinuousScrapeOrchestrator, ws = self._import_orch_classes()
self._setup_ws_data_dir()
try:
orch = ContinuousScrapeOrchestrator()
aid = make_account_id()
create_account(TEST_DATA, aid, continuous_scraping={
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
})
orch.start_all()
snap = orch.snapshot_for(aid)
assert snap["status"]["running"] is False
finally:
self._restore_ws_data_dir()
class TestMessageSearch:
"""load_messages() SQL correctness with pagination and search."""
def _import_load_messages(self):
import webui_server as ws_module
# Point at TEST_DATA
self._ws_orig = ws_module.DATA_DIR
ws_module.DATA_DIR = TEST_DATA
return ws_module.load_messages
def _restore(self):
import webui_server as ws_module
ws_module.DATA_DIR = self._ws_orig
def test_search_filters_correctly(self):
channel_id = "-100searchtest"
messages = [
{"message_id": 1, "date": "2024-01-01", "message": "hello world"},
{"message_id": 2, "date": "2024-01-02", "message": "foo bar baz"},
{"message_id": 3, "date": "2024-01-03", "message": "hello again"},
]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=100, search="hello")
finally:
self._restore()
texts = [m["text"] for m in result]
assert "hello world" in texts
assert "hello again" in texts
assert "foo bar baz" not in texts
def test_before_id_pagination_works(self):
channel_id = "-100pagination"
messages = [
{"message_id": i, "date": f"2024-01-{i:02d}", "message": f"msg-{i}"}
for i in range(1, 21)
]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=5, before_message_id=15)
finally:
self._restore()
assert len(result) == 5
ids = [m["message_id"] for m in result]
assert all(i < 15 for i in ids)
# load_messages returns rows in ascending message_id order (reversed from DESC query)
assert ids == [10, 11, 12, 13, 14]
def test_search_with_pagination(self):
channel_id = "-100searchpages"
messages = [
{"message_id": i, "date": f"2024-01-{i:02d}", "message": f"hello-{i}" if i % 2 else f"other-{i}"}
for i in range(1, 21)
]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=3, before_message_id=15, search="hello")
finally:
self._restore()
# IDs matching "hello-" below 15: 13, 11, 9, 7, 5, 3, 1 → LIMIT 3 → [9, 11, 13] (ascending)
ids = [m["message_id"] for m in result]
assert ids == [9, 11, 13], f"got {ids}"
assert all("hello" in m["text"] for m in result)
def test_empty_result_no_error(self):
channel_id = "-100emptysearch"
messages = [{"message_id": 1, "date": "2024-01-01", "message": "only one"}]
create_channel_db(TEST_DATA, "default", channel_id, messages)
load_messages = self._import_load_messages()
try:
result = load_messages("default", channel_id, limit=100, search="nonexistent")
finally:
self._restore()
assert result == []
class TestHealthPayload:
"""Health endpoint structure."""
def test_health_payload_structure(self):
aid = make_account_id()
create_account(TEST_DATA, aid)
payload = health_payload(
TEST_DATA,
TEST_SESSION,
get_global_store(TEST_DATA),
{},
0,
[aid],
)
assert "ok" in payload
assert "status" in payload
assert "checks" in payload
checks = payload["checks"]
assert "data_dir" in checks
assert "session_dir" in checks
assert "accounts" in checks
assert "items" in checks["accounts"]
assert aid in checks["accounts"]["items"]
def test_health_accounts_wrapper(self):
a1, a2 = make_account_id(), make_account_id()
create_account(TEST_DATA, a1)
create_account(TEST_DATA, a2)
payload = health_payload(
TEST_DATA, TEST_SESSION,
get_global_store(TEST_DATA),
{}, 0, [a1, a2],
)
acc_checks = payload["checks"]["accounts"]
assert acc_checks["ok"] is True
assert len(acc_checks["items"]) == 2
class TestPersistence:
"""UI persistence helpers."""
def test_active_account_localstorage(self):
"""Verifies the save/load logic pattern used in app.js."""
accounts = [
{"id": "work", "label": "Work"},
{"id": "home", "label": "Home"},
]
saved_id = "work"
loaded = None
if saved_id and any(a["id"] == saved_id for a in accounts):
loaded = saved_id
if not loaded:
loaded = next((a["id"] for a in accounts), None)
assert loaded == "work"
saved_id = "longgone"
loaded = None
if saved_id and any(a["id"] == saved_id for a in accounts):
loaded = saved_id
if not loaded and accounts:
loaded = accounts[0]["id"]
assert loaded == "work"
def test_viewer_fallback_on_missing_account(self):
"""Viewer's loadViewerAccount fallback logic."""
accounts = [
{"id": "alpha", "label": "Alpha"},
{"id": "beta", "label": "Beta"},
]
requested = "nonexistent"
requested_exists = requested and any(a["id"] == requested for a in accounts)
account_id = requested if requested_exists else (accounts[0]["id"] if accounts else None)
assert account_id == "alpha"
account_id2 = requested if (requested and any(a["id"] == requested for a in [])) else None
assert account_id2 is None
class TestJobDeduplicationSchema:
"""JobRunner create_job dedup logic."""
def _import_job_runner(self):
import webui_server as ws_module
return ws_module.JobRunner
def test_create_job_dedup_by_account(self):
JobRunner = self._import_job_runner()
runner = JobRunner()
runner.queue.put = lambda job: None
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
assert j1.job_id == j2.job_id, "duplicate job was created instead of being reused"
assert "Reused existing active job" in j2.logs
runner.shutdown(timeout=0)
def test_create_job_allows_different_accounts(self):
JobRunner = self._import_job_runner()
runner = JobRunner()
runner.queue.put = lambda job: None
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
time.sleep(0.002) # ensure different timestamp -> different job_id
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc2"})
assert j1.job_id != j2.job_id
runner.shutdown(timeout=0)
def test_create_job_allows_after_previous_completes(self):
JobRunner = self._import_job_runner()
runner = JobRunner()
runner.queue.put = lambda job: None
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
j1.status = "done"
time.sleep(0.002) # ensure different timestamp -> different job_id
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
assert j1.job_id != j2.job_id
runner.shutdown(timeout=0)
class TestAccountHealthSummary:
"""account_health_summary structure."""
def _import_health_summary(self):
import webui_server as ws_module
self._ws_orig_data = ws_module.DATA_DIR
self._ws_orig_session = ws_module.SESSION_DIR
ws_module.DATA_DIR = TEST_DATA
ws_module.SESSION_DIR = TEST_SESSION
return ws_module.account_health_summary, ws_module.JobRunner, ws_module
def test_health_summary_structure(self):
account_health_summary, JobRunner, ws = self._import_health_summary()
aid = make_account_id()
create_account(TEST_DATA, aid, api_id=1, api_hash="x")
try:
runner = JobRunner()
health = account_health_summary(aid, runner)
finally:
ws.DATA_DIR = self._ws_orig_data
ws.SESSION_DIR = self._ws_orig_session
assert health["account_id"] == aid
assert "label" in health
assert "data_dir_exists" in health
assert "session_ready" in health
assert "api_credentials" in health
assert health["api_credentials"] is True
assert "channel_count" in health
assert "message_count" in health
assert "media_count" in health
# ── Cleanup all temp data ──────────────────────────────────────────────────
def cleanup_test_data():
if TEST_TMP.exists():
shutil.rmtree(str(TEST_TMP), ignore_errors=True)
import atexit # noqa: E402
atexit.register(cleanup_test_data) # noqa: E402
+1046
View File
File diff suppressed because it is too large Load Diff
+377
View File
@@ -0,0 +1,377 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Control Panel</title>
<link rel="stylesheet" href="/static/style.css?v=8" />
</head>
<body>
<div class="app-shell">
<aside class="sidebar">
<div class="brand-block">
<svg class="brand-icon" viewBox="0 0 24 24" aria-hidden="true">
<path d="m21 3-7.6 18-3.8-7.6L2 9.6zM9.6 13.4 14 10" />
</svg>
<div class="eyebrow">Telegram Scraper</div>
<h1>Control</h1>
<p class="muted">Local scraper console for channels, jobs, and continuous runs.</p>
</div>
<nav class="nav-links">
<a class="nav-link active" href="/"
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true">
<rect x="4" y="4" width="6" height="6" />
<rect x="14" y="4" width="6" height="6" />
<rect x="4" y="14" width="6" height="6" />
<rect x="14" y="14" width="6" height="6" /></svg
>Dashboard</a
>
<a class="nav-link" href="/settings"
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true">
<circle cx="12" cy="12" r="3" />
<path
d="M12 2v3m0 14v3M2 12h3m14 0h3m-2.9-7.1-2.1 2.1M4.9 19.1 7 17m0-10-2.1-2.1m12.2 14.2-2.1-2.1" /></svg
>Settings</a
>
<a class="nav-link" href="/viewer"
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true"><path d="M4 5h16v12H8l-4 3z" /></svg>Message
Viewer</a
>
<a class="nav-link" href="/swagger"
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true">
<path d="m8 5-5 7 5 7m8-14 5 7-5 7M14 3l-4 18" /></svg
>API Docs</a
>
<a class="nav-link" href="/health"
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true"><path d="M3 12h4l2-6 4 12 2-6h6" /></svg
>Health</a
>
</nav>
<section class="status-card" id="active-account-card">
<div class="section-title">Active Account</div>
<div id="active-account-name" class="status-badge">Loading...</div>
<p id="active-account-status" class="muted small"></p>
</section>
<section class="status-card">
<div class="section-title">Actions</div>
<button class="button primary" id="scrape-all-btn">Scrape all channels</button>
<button class="button" id="export-all-btn">Export all data</button>
<button class="button" id="refresh-dialogs-btn">Refresh dialogs</button>
<a class="button" id="open-settings-btn" href="/settings">Settings</a>
</section>
</aside>
<main class="content">
<header class="dashboard-topbar">
<div class="dashboard-title"><span class="menu-icon" aria-hidden="true"></span>Dashboard</div>
<div class="runtime-status">
<span class="runtime-dot"></span>Service: Running <i></i> Scraper: Idle <i></i
><span class="runtime-clock"></span>
</div>
</header>
<!-- Account Tabs -->
<div id="account-tabs" class="account-tabs"></div>
<!-- Account Dashboard Panels -->
<div id="account-panels">
<!-- Each account's dashboard is rendered here dynamically -->
</div>
<!-- Fallback for no accounts -->
<section id="no-accounts-msg" class="panel hidden">
<div class="panel-header">
<div>
<h2>No Accounts Configured</h2>
<p class="muted">
Click "Add Account" in Settings to get started, or use the legacy single-account mode.
</p>
</div>
</div>
</section>
</main>
</div>
<!-- ════════ Settings Dialog ════════ -->
<dialog id="settings-dialog" class="settings-dialog">
<div class="dialog-shell">
<div class="dialog-header">
<div>
<div class="eyebrow">Settings</div>
<h2>Global & Accounts</h2>
</div>
<button class="button button-small" id="close-settings-btn" type="button">Close</button>
</div>
<!-- ── Accounts Management ── -->
<section class="settings-section">
<div class="section-title">Accounts</div>
<div id="accounts-list" class="accounts-list"></div>
<form id="add-account-form" class="stack-form add-account-form">
<input id="add-account-id" name="account_id" placeholder="Account ID (e.g. work)" required />
<input id="add-account-label" name="label" placeholder="Display name (e.g. Work Account)" />
<input id="add-account-api-id" name="api_id" placeholder="API ID" />
<input id="add-account-api-hash" name="api_hash" placeholder="API Hash" />
<button class="button primary" type="submit">Add Account</button>
</form>
<label class="import-account-row">
<span class="muted small">Import account settings JSON</span>
<input id="import-account-file" type="file" accept="application/json,.json" />
</label>
</section>
<!-- ── Account Auth ── (shown per account selected in UI) -->
<section class="settings-section" id="account-auth-section">
<div class="section-title">Account Auth: <span id="auth-account-label">-</span></div>
<form id="credentials-form" class="stack-form">
<input id="api-id-input" name="api_id" placeholder="API ID" />
<input id="api-hash-input" name="api_hash" placeholder="API Hash" />
<button class="button" type="submit">Save credentials</button>
</form>
<div class="auth-actions">
<button class="button primary" id="start-qr-btn" type="button">Start QR login</button>
</div>
<div id="qr-wrap" class="qr-wrap hidden">
<img id="qr-image" alt="Telegram QR login" />
<p class="muted small">Open Telegram -> Settings -> Devices -> Scan QR.</p>
</div>
<form id="phone-form" class="stack-form">
<input id="phone-input" name="phone" placeholder="+1234567890" />
<button class="button" type="submit">Send code</button>
</form>
<form id="code-form" class="stack-form hidden">
<input id="code-input" name="code" placeholder="Telegram code" />
<button class="button" type="submit">Confirm code</button>
</form>
<form id="password-form" class="stack-form hidden">
<input id="password-input" name="password" type="password" placeholder="2FA password" />
<button class="button" type="submit">Confirm password</button>
</form>
</section>
<!-- ── Scraping ── -->
<section class="settings-section">
<div class="section-title">Scraping</div>
<label class="toggle-row">
<span>Download media</span>
<span class="switch">
<input id="scrape-media-toggle" type="checkbox" />
<span class="switch-slider"></span>
</span>
</label>
</section>
</div>
</dialog>
<!-- ════════ Templates ════════ -->
<!-- Account Tab -->
<template id="account-tab-template">
<button class="account-tab" type="button">
<span class="account-tab-name"></span>
<span class="account-tab-status"></span>
</button>
</template>
<!-- Account Dashboard Panel -->
<template id="account-panel-template">
<section class="account-panel">
<!-- Dashboard tabs: Overview, Continuous -->
<div class="content-tabs">
<button class="content-tab active" data-tab-target="overview" type="button">Overview</button>
<button class="content-tab" data-tab-target="continuous" type="button">Continuous</button>
</div>
<!-- Overview Tab -->
<div class="tab-panel active" data-panel="overview">
<section class="panel-grid">
<div class="panel stat-panel">
<div class="section-title">Channels</div>
<div class="stat-value channel-count">0</div>
<div class="stat-caption">Tracked channels</div>
<div class="stat-meta">
<span><strong class="channel-active-count">0</strong> Active</span
><span><strong class="channel-inactive-count">0</strong> Inactive</span>
</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Media scraping</div>
<div class="stat-value stat-compact scrape-media-label">OFF</div>
<div class="stat-caption">Items downloaded</div>
<div class="stat-meta">
<span><strong class="media-session-count">0</strong> This session</span
><span><strong class="media-total-count">0</strong> Total</span>
</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Forwarding rules</div>
<div class="stat-value forwarding-count">0</div>
<div class="stat-caption">Active rules</div>
<div class="stat-meta">
<span><strong class="forwarding-active-count">0</strong> Active</span
><span><strong>0</strong> Paused</span>
</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Account health</div>
<div class="stat-value stat-compact account-health-label">-</div>
<div class="stat-caption">Session status</div>
<div class="stat-meta">
<span><strong class="health-session-value">-</strong> Session</span
><span><strong class="health-rate-value">-</strong> Rate limit</span>
</div>
<div class="muted small account-health-detail"></div>
</div>
</section>
<section class="panel">
<div class="panel-header">
<div>
<h2>Tracked Channels</h2>
<p class="muted">Per-account channel list.</p>
</div>
<form class="inline-form add-channel-form">
<input class="add-channel-id" name="channel_id" placeholder="ID or @username" required />
<input class="add-channel-name" name="name" placeholder="Display name" />
<button class="button primary" type="submit">Add</button>
</form>
</div>
<div class="table-wrap">
<table>
<thead>
<tr>
<th>Channel</th>
<th>Messages</th>
<th>Media</th>
<th>Last message</th>
<th>Actions</th>
</tr>
</thead>
<tbody class="channels-table"></tbody>
</table>
</div>
</section>
<section class="panel jobs-panel">
<div class="panel-header">
<div>
<h2>Jobs</h2>
<p class="muted">Queued work runs one job at a time.</p>
</div>
<button class="button refresh-jobs-btn">Refresh</button>
</div>
<div class="jobs-list"></div>
</section>
</div>
<!-- Continuous Tab -->
<div class="tab-panel" data-panel="continuous">
<section class="panel">
<div class="panel-header">
<div>
<h2>Continuous Scraping</h2>
<p class="muted">Per-account continuous scrape settings.</p>
</div>
</div>
<form class="continuous-form">
<label class="toggle-row">
<span>Enabled</span>
<span class="switch">
<input type="checkbox" class="continuous-enabled" checked />
<span class="switch-slider"></span>
</span>
</label>
<label class="stack-form">
<span class="muted small">Interval, minutes</span>
<input type="number" class="continuous-interval" min="1" value="1" />
</label>
<label class="toggle-row">
<span>All tracked channels</span>
<span class="switch">
<input type="checkbox" class="continuous-all" checked />
<span class="switch-slider"></span>
</span>
</label>
<div class="channel-picker">
<div class="muted small">Continuous channel set</div>
<div class="checkbox-grid continuous-channel-list"></div>
</div>
<button class="button primary" type="submit">Save continuous settings</button>
</form>
<div class="continuous-meta">
<div>Status: <span class="continuous-status">-</span></div>
<div>Last run: <span class="continuous-last-iteration">-</span></div>
<div>Next run: <span class="continuous-next-run">-</span></div>
<div>Last error: <span class="continuous-last-error">-</span></div>
</div>
<div class="log-viewer continuous-logs"></div>
</section>
</div>
</section>
</template>
<!-- Channel Row -->
<template id="channel-row-template">
<tr>
<td>
<div class="channel-name"></div>
<div class="muted small channel-id"></div>
</td>
<td class="message-count"></td>
<td class="media-count"></td>
<td class="last-date"></td>
<td>
<div class="action-row">
<button class="button button-small scrape-btn">Scrape</button>
<button class="button button-small export-btn">Export</button>
<button class="button button-small export-view-btn">Viewer</button>
<button class="button button-small media-btn">Rescrape media</button>
<button class="button button-small danger remove-btn">Remove</button>
</div>
</td>
</tr>
</template>
<!-- Job Template -->
<template id="job-template">
<article class="job-card">
<div class="job-head">
<div class="job-title"></div>
<div class="job-status"></div>
</div>
<div class="muted small job-time"></div>
<pre class="job-logs"></pre>
</article>
</template>
<!-- Account list item in settings -->
<template id="account-list-item-template">
<div class="account-list-item">
<div class="account-list-info">
<span class="account-list-label"></span>
<span class="muted small account-list-id"></span>
<span class="account-list-auth-status"></span>
</div>
<div class="account-list-actions">
<button class="button button-small account-select-btn">Select</button>
<button class="button button-small account-export-btn">Export</button>
<button class="button button-small danger account-remove-btn">Remove</button>
</div>
</div>
</template>
<script src="/static/app.js?v=4"></script>
</body>
</html>
+156
View File
@@ -0,0 +1,156 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Settings</title>
<link rel="stylesheet" href="/static/style.css?v=8" />
</head>
<body>
<div class="app-shell settings-page-shell">
<aside class="sidebar">
<div class="brand-block">
<div class="eyebrow">Telegram Scraper</div>
<h1>Settings</h1>
<p class="muted">Account credentials, import/export, and runtime safety controls.</p>
</div>
<nav class="nav-links">
<a class="nav-link" href="/">Dashboard</a>
<a class="nav-link active" href="/settings">Settings</a>
<a class="nav-link" href="/viewer">Message Viewer</a>
<a class="nav-link" href="/swagger">API Docs</a>
<a class="nav-link" href="/health">Health</a>
</nav>
<section class="status-card">
<div class="section-title">Accounts</div>
<div id="settings-account-list" class="settings-account-list"></div>
</section>
</aside>
<main class="content settings-page">
<section class="panel settings-hero">
<div>
<div class="eyebrow">Account Settings</div>
<h2 id="settings-title">Loading accounts</h2>
<p id="settings-subtitle" class="muted">Select an account to manage credentials and scraper options.</p>
</div>
<div class="action-row">
<button class="button" id="settings-export-btn" type="button" disabled>Export</button>
<button class="button danger" id="settings-delete-btn" type="button" disabled>Delete</button>
</div>
</section>
<section id="settings-empty" class="panel hidden">
<div class="empty-state">
<h2>Add first account</h2>
<p class="muted">Create an account below or import an exported account JSON.</p>
</div>
</section>
<section class="panel-grid settings-health-grid">
<div class="panel stat-panel">
<div class="section-title">Credentials</div>
<div class="stat-value stat-compact" id="health-credentials">-</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Session</div>
<div class="stat-value stat-compact" id="health-session">-</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Continuous</div>
<div class="stat-value stat-compact" id="health-continuous">-</div>
</div>
<div class="panel stat-panel">
<div class="section-title">Last scrape</div>
<div class="stat-value stat-compact" id="health-last-scrape">-</div>
</div>
</section>
<section class="settings-layout">
<div class="panel">
<div class="panel-header">
<div>
<h2>Create or Import</h2>
<p class="muted">New accounts start without continuous scraping enabled.</p>
</div>
</div>
<form id="settings-add-account-form" class="stack-form">
<input id="settings-add-account-id" name="account_id" placeholder="Account ID (e.g. work)" required />
<input id="settings-add-account-label" name="label" placeholder="Display name" />
<input id="settings-add-account-api-id" name="api_id" placeholder="API ID" />
<input id="settings-add-account-api-hash" name="api_hash" placeholder="API Hash" />
<button class="button primary" type="submit">Add Account</button>
</form>
<label class="import-account-row">
<span class="muted small">Import account settings JSON</span>
<input id="settings-import-account-file" type="file" accept="application/json,.json" />
</label>
</div>
<div class="panel">
<div class="panel-header">
<div>
<h2>Credentials & Auth</h2>
<p class="muted" id="settings-auth-label">No account selected.</p>
</div>
</div>
<form id="settings-credentials-form" class="stack-form">
<input id="settings-api-id-input" name="api_id" placeholder="API ID" />
<input id="settings-api-hash-input" name="api_hash" placeholder="API Hash" />
<button class="button" type="submit">Save credentials</button>
</form>
<div class="auth-actions">
<button class="button primary" id="settings-start-qr-btn" type="button">Start QR login</button>
</div>
<div id="settings-qr-wrap" class="qr-wrap hidden">
<img id="settings-qr-image" alt="Telegram QR login" />
<p class="muted small">Open Telegram -> Settings -> Devices -> Scan QR.</p>
</div>
<form id="settings-phone-form" class="stack-form">
<input id="settings-phone-input" name="phone" placeholder="+1234567890" />
<button class="button" type="submit">Send code</button>
</form>
<form id="settings-code-form" class="stack-form hidden">
<input id="settings-code-input" name="code" placeholder="Telegram code" />
<button class="button" type="submit">Confirm code</button>
</form>
<form id="settings-password-form" class="stack-form hidden">
<input id="settings-password-input" name="password" type="password" placeholder="2FA password" />
<button class="button" type="submit">Confirm password</button>
</form>
</div>
<div class="panel">
<div class="panel-header">
<div>
<h2>Scraping</h2>
<p class="muted">Account-level parser settings.</p>
</div>
</div>
<label class="toggle-row">
<span>Download media</span>
<span class="switch">
<input id="settings-scrape-media-toggle" type="checkbox" />
<span class="switch-slider"></span>
</span>
</label>
<div class="settings-meta-list">
<div>Tracked channels: <span id="settings-channel-count">0</span></div>
<div>Messages: <span id="settings-message-count">0</span></div>
<div>Media: <span id="settings-media-count">0</span></div>
</div>
</div>
</section>
</main>
</div>
<script src="/static/settings.js?v=1"></script>
</body>
</html>
+431
View File
@@ -0,0 +1,431 @@
const settingsState = {
accounts: [],
activeAccount: null,
dashboard: null,
auth: null,
continuous: null,
};
const ACTIVE_ACCOUNT_STORAGE_KEY = 'telegramScraper.activeAccount';
async function api(path, options = {}) {
const response = await fetch(path, {
headers: { 'Content-Type': 'application/json' },
...options,
});
const data = await response.json();
if (!response.ok) {
throw new Error(data.error || 'Request failed');
}
return data;
}
function showToast(message, type = 'info') {
let root = document.getElementById('toast-root');
if (!root) {
root = document.createElement('div');
root.id = 'toast-root';
root.className = 'toast-root';
document.body.appendChild(root);
}
const toast = document.createElement('div');
toast.className = `toast toast-${type}`;
toast.textContent = message;
root.appendChild(toast);
window.setTimeout(() => {
toast.classList.add('toast-hide');
window.setTimeout(() => toast.remove(), 250);
}, 3600);
}
function downloadJson(filename, payload) {
const blob = new Blob([JSON.stringify(payload, null, 2) + '\n'], {
type: 'application/json',
});
const url = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
link.download = filename;
document.body.appendChild(link);
link.click();
link.remove();
URL.revokeObjectURL(url);
}
function displayTime(value) {
if (!value) return '-';
const date = new Date(value);
if (Number.isNaN(date.getTime())) return String(value).replace('T', ' ').replace('+00:00', ' UTC');
return date.toLocaleString(undefined, {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
});
}
function isAccountAuthorized(auth) {
const status = auth?.auth_status || auth || {};
return (
auth?.phase === 'authorized' ||
auth?.status === 'authorized' ||
status.status === 'ready' ||
status.status === 'authorized'
);
}
function accountLabel(account) {
return account?.label || account?.id || 'No account';
}
function setActiveAccount(accountId) {
settingsState.activeAccount = accountId || null;
if (accountId) {
localStorage.setItem(ACTIVE_ACCOUNT_STORAGE_KEY, accountId);
} else {
localStorage.removeItem(ACTIVE_ACCOUNT_STORAGE_KEY);
}
}
function chooseInitialAccount() {
const requested = new URLSearchParams(window.location.search).get('account');
if (requested && settingsState.accounts.some((account) => account.id === requested)) {
return requested;
}
const saved = localStorage.getItem(ACTIVE_ACCOUNT_STORAGE_KEY);
if (saved && settingsState.accounts.some((account) => account.id === saved)) {
return saved;
}
return settingsState.accounts[0]?.id || null;
}
function renderAccountList() {
const root = document.getElementById('settings-account-list');
root.innerHTML = '';
if (!settingsState.accounts.length) {
root.innerHTML = '<div class="empty-state">No accounts yet.</div>';
return;
}
settingsState.accounts.forEach((account) => {
const button = document.createElement('button');
button.className = 'settings-account-button';
button.type = 'button';
button.classList.toggle('active', account.id === settingsState.activeAccount);
const name = document.createElement('span');
name.className = 'settings-account-name';
name.textContent = accountLabel(account);
const status = document.createElement('span');
status.className = isAccountAuthorized(account.auth) ? 'settings-account-status ok' : 'settings-account-status';
status.textContent = isAccountAuthorized(account.auth) ? 'Authorized' : 'Needs auth';
button.append(name, status);
button.addEventListener('click', () => loadAccount(account.id));
root.appendChild(button);
});
}
function renderEmptyState() {
const hasAccount = Boolean(settingsState.activeAccount);
document.getElementById('settings-empty').classList.toggle('hidden', hasAccount);
document.getElementById('settings-export-btn').disabled = !hasAccount;
document.getElementById('settings-delete-btn').disabled = !hasAccount;
document
.getElementById('settings-credentials-form')
.querySelectorAll('input, button')
.forEach((node) => {
node.disabled = !hasAccount;
});
document.getElementById('settings-start-qr-btn').disabled = !hasAccount;
document
.getElementById('settings-phone-form')
.querySelectorAll('input, button')
.forEach((node) => {
node.disabled = !hasAccount;
});
document.getElementById('settings-scrape-media-toggle').disabled = !hasAccount;
}
function renderAccountData() {
const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount);
const dashboard = settingsState.dashboard || {};
const health = dashboard.health || {};
const auth = settingsState.auth || account?.auth || {};
const continuous = settingsState.continuous || {};
const continuousStatus = continuous.status || {};
const continuousConfig = continuous.config || {};
document.getElementById('settings-title').textContent = account ? accountLabel(account) : 'No account selected';
document.getElementById('settings-subtitle').textContent = account
? `Account ID: ${account.id}`
: 'Create or import an account to manage settings.';
document.getElementById('settings-auth-label').textContent = account
? `${accountLabel(account)} auth status`
: 'No account selected.';
document.getElementById('health-credentials').textContent = health.api_credentials ? 'Saved' : 'Missing';
document.getElementById('health-credentials').style.color = health.api_credentials ? 'var(--ok)' : 'var(--warn)';
document.getElementById('health-session').textContent =
health.session_ready || isAccountAuthorized(auth) ? 'Ready' : 'Missing';
document.getElementById('health-session').style.color =
health.session_ready || isAccountAuthorized(auth) ? 'var(--ok)' : 'var(--warn)';
document.getElementById('health-continuous').textContent = continuousStatus.running
? 'Running'
: continuousConfig.enabled
? 'Enabled'
: 'Stopped';
document.getElementById('health-continuous').style.color = continuousStatus.running ? 'var(--ok)' : 'var(--dim)';
document.getElementById('health-last-scrape').textContent = displayTime(
health.last_scrape || continuousStatus.last_iteration_at,
);
document.getElementById('settings-scrape-media-toggle').checked = Boolean(dashboard.dashboard?.scrape_media);
document.getElementById('settings-channel-count').textContent = String(health.channel_count || 0);
document.getElementById('settings-message-count').textContent = String(health.message_count || 0);
document.getElementById('settings-media-count').textContent = String(health.media_count || 0);
renderEmptyState();
renderAccountList();
}
async function loadAccounts() {
const payload = await api('/api/accounts');
settingsState.accounts = payload.accounts || [];
setActiveAccount(chooseInitialAccount());
renderAccountList();
if (settingsState.activeAccount) {
await loadAccount(settingsState.activeAccount, { preserveUrl: true });
} else {
settingsState.dashboard = null;
settingsState.auth = null;
settingsState.continuous = null;
renderAccountData();
}
}
async function loadAccount(accountId, options = {}) {
setActiveAccount(accountId);
const [dashboard, auth, continuous] = await Promise.all([
api(`/api/accounts/${encodeURIComponent(accountId)}`),
api(`/api/accounts/${encodeURIComponent(accountId)}/auth`).catch(() => ({})),
api(`/api/accounts/${encodeURIComponent(accountId)}/continuous`).catch(() => ({})),
]);
settingsState.dashboard = dashboard;
settingsState.auth = auth;
settingsState.continuous = continuous;
if (!options.preserveUrl) {
history.replaceState(null, '', `/settings?account=${encodeURIComponent(accountId)}`);
}
renderAccountData();
}
async function createAccount(event) {
event.preventDefault();
const accountId = document.getElementById('settings-add-account-id').value.trim();
const label = document.getElementById('settings-add-account-label').value.trim();
const apiId = document.getElementById('settings-add-account-api-id').value.trim();
const apiHash = document.getElementById('settings-add-account-api-hash').value.trim();
if (!accountId) return;
await api('/api/accounts', {
method: 'POST',
body: JSON.stringify({
account_id: accountId,
label: label || accountId,
api_id: apiId ? parseInt(apiId) : undefined,
api_hash: apiHash || undefined,
}),
});
event.currentTarget.reset();
await loadAccounts();
await loadAccount(accountId);
showToast(`Added ${accountId}.`, 'success');
}
async function importAccount(event) {
const file = event.currentTarget.files?.[0];
if (!file) return;
try {
const payload = JSON.parse(await file.text());
const accountId = payload.account_id || payload.id;
if (!accountId) throw new Error('account_id is missing in import file');
await api('/api/accounts/import', {
method: 'POST',
body: JSON.stringify(payload),
});
event.currentTarget.value = '';
await loadAccounts();
await loadAccount(accountId);
showToast(`Imported ${accountId}.`, 'success');
} catch (err) {
showToast(`Failed to import account: ${err.message}`, 'error');
}
}
async function saveCredentials(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const apiId = document.getElementById('settings-api-id-input').value.trim();
const apiHash = document.getElementById('settings-api-hash-input').value.trim();
if (!apiId || !apiHash) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/credentials`, {
method: 'POST',
body: JSON.stringify({ api_id: apiId, api_hash: apiHash }),
});
await loadAccount(settingsState.activeAccount);
showToast('Credentials saved.', 'success');
}
async function startQrLogin() {
if (!settingsState.activeAccount) return;
const data = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/qr/start`, {
method: 'POST',
body: JSON.stringify({}),
});
if (data.qr_image) {
document.getElementById('settings-qr-image').src = data.qr_image;
document.getElementById('settings-qr-wrap').classList.remove('hidden');
}
await loadAccount(settingsState.activeAccount);
}
async function requestPhoneCode(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const phone = document.getElementById('settings-phone-input').value.trim();
if (!phone) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/request`, {
method: 'POST',
body: JSON.stringify({ phone }),
});
document.getElementById('settings-code-form').classList.remove('hidden');
await loadAccount(settingsState.activeAccount);
}
async function submitPhoneCode(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const code = document.getElementById('settings-code-input').value.trim();
if (!code) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/submit`, {
method: 'POST',
body: JSON.stringify({ code }),
});
document.getElementById('settings-code-input').value = '';
await loadAccount(settingsState.activeAccount);
}
async function submitPassword(event) {
event.preventDefault();
if (!settingsState.activeAccount) return;
const password = document.getElementById('settings-password-input').value.trim();
if (!password) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/password`, {
method: 'POST',
body: JSON.stringify({ password }),
});
document.getElementById('settings-password-input').value = '';
await loadAccount(settingsState.activeAccount);
}
async function exportAccount() {
if (!settingsState.activeAccount) return;
const payload = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/export`);
downloadJson(`telegram-scraper-account-${settingsState.activeAccount}.json`, payload);
showToast('Account exported.', 'success');
}
async function deleteAccount() {
if (!settingsState.activeAccount) return;
const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount);
if (!window.confirm(`Remove account "${accountLabel(account)}"? All its account data will be deleted.`)) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}`, { method: 'DELETE' });
setActiveAccount(null);
await loadAccounts();
showToast('Account deleted.', 'success');
}
async function toggleMedia(event) {
if (!settingsState.activeAccount) return;
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/settings/media`, {
method: 'POST',
body: JSON.stringify({ value: event.currentTarget.checked }),
});
await loadAccount(settingsState.activeAccount);
}
function bindEvents() {
document.getElementById('settings-add-account-form').addEventListener('submit', async (event) => {
try {
await createAccount(event);
} catch (err) {
showToast(`Failed to add account: ${err.message}`, 'error');
}
});
document.getElementById('settings-import-account-file').addEventListener('change', importAccount);
document.getElementById('settings-credentials-form').addEventListener('submit', async (event) => {
try {
await saveCredentials(event);
} catch (err) {
showToast(`Failed to save credentials: ${err.message}`, 'error');
}
});
document.getElementById('settings-start-qr-btn').addEventListener('click', async () => {
try {
await startQrLogin();
} catch (err) {
showToast(`QR login failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-phone-form').addEventListener('submit', async (event) => {
try {
await requestPhoneCode(event);
} catch (err) {
showToast(`Phone login failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-code-form').addEventListener('submit', async (event) => {
try {
await submitPhoneCode(event);
} catch (err) {
showToast(`Code submit failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-password-form').addEventListener('submit', async (event) => {
try {
await submitPassword(event);
} catch (err) {
showToast(`Password submit failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-export-btn').addEventListener('click', async () => {
try {
await exportAccount();
} catch (err) {
showToast(`Export failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-delete-btn').addEventListener('click', async () => {
try {
await deleteAccount();
} catch (err) {
showToast(`Delete failed: ${err.message}`, 'error');
}
});
document.getElementById('settings-scrape-media-toggle').addEventListener('change', async (event) => {
try {
await toggleMedia(event);
} catch (err) {
showToast(`Media setting failed: ${err.message}`, 'error');
}
});
}
bindEvents();
loadAccounts().catch((err) => {
console.error(err);
showToast(`Failed to load settings: ${err.message}`, 'error');
});
+2224
View File
File diff suppressed because it is too large Load Diff
+58
View File
@@ -0,0 +1,58 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper API</title>
<link rel="stylesheet" href="/static/style.css?v=6" />
</head>
<body>
<div class="viewer-shell">
<aside class="viewer-sidebar">
<div class="viewer-sidebar-head">
<div>
<div class="eyebrow">OpenAPI</div>
<h1>API Docs</h1>
</div>
<a class="button button-small" href="/">Dashboard</a>
</div>
<nav class="nav-links">
<a class="nav-link" href="/openapi.json">openapi.json</a>
<a class="nav-link" href="/health">Health</a>
<a class="nav-link" href="/viewer">Message Viewer</a>
</nav>
</aside>
<main class="viewer-main">
<header class="viewer-header">
<div>
<h2>Endpoints</h2>
<p class="muted">Local API surface exposed by the web UI server.</p>
</div>
</header>
<section id="api-docs" class="api-docs"></section>
</main>
</div>
<template id="api-section-template">
<section class="panel api-section">
<h2 class="api-path"></h2>
<div class="api-methods"></div>
</section>
</template>
<template id="api-method-template">
<article class="api-method">
<div class="api-method-head">
<span class="api-verb"></span>
<span class="api-summary"></span>
</div>
<p class="muted api-description"></p>
<pre class="job-logs api-body"></pre>
</article>
</template>
<script src="/static/swagger.js"></script>
</body>
</html>
+53
View File
@@ -0,0 +1,53 @@
async function loadSpec() {
const response = await fetch('/openapi.json');
const spec = await response.json();
if (!response.ok) {
throw new Error(spec.error || 'Failed to load OpenAPI spec');
}
return spec;
}
function methodPayload(operation) {
const body = operation.requestBody?.content?.['application/json']?.schema;
if (!body) return '';
return JSON.stringify(body.example || body.properties || body, null, 2);
}
function renderSpec(spec) {
const root = document.getElementById('api-docs');
const sectionTemplate = document.getElementById('api-section-template');
const methodTemplate = document.getElementById('api-method-template');
root.innerHTML = '';
Object.entries(spec.paths || {}).forEach(([path, methods]) => {
const section = sectionTemplate.content.firstElementChild.cloneNode(true);
section.querySelector('.api-path').textContent = path;
const methodsRoot = section.querySelector('.api-methods');
Object.entries(methods).forEach(([method, operation]) => {
const node = methodTemplate.content.firstElementChild.cloneNode(true);
node.querySelector('.api-verb').textContent = method.toUpperCase();
node.querySelector('.api-summary').textContent = operation.summary || '';
node.querySelector('.api-description').textContent = operation.description || '';
const body = methodPayload(operation);
const bodyNode = node.querySelector('.api-body');
if (body) {
bodyNode.textContent = body;
} else {
bodyNode.remove();
}
methodsRoot.appendChild(node);
});
root.appendChild(section);
});
}
loadSpec()
.then(renderSpec)
.catch((error) => {
console.error(error);
document.getElementById('api-docs').innerHTML = `<section class="panel">${error.message}</section>`;
});
+85
View File
@@ -0,0 +1,85 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Viewer</title>
<link rel="stylesheet" href="/static/style.css?v=8" />
</head>
<body>
<div class="viewer-shell">
<aside class="viewer-sidebar">
<div class="viewer-sidebar-head">
<div>
<div class="eyebrow">Export Viewer</div>
<h1>Messages</h1>
<p class="muted small">
Account:
<select id="viewer-account-select"></select>
</p>
</div>
<button id="sidebar-toggle" class="button button-small sidebar-toggle"></button>
<a class="button button-small" href="/">Dashboard</a>
</div>
<div id="viewer-channel-list" class="viewer-channel-list"></div>
</aside>
<main class="viewer-main">
<header class="viewer-header">
<div>
<h2 id="viewer-title">Select a channel</h2>
<p id="viewer-subtitle" class="muted">Reading messages from the local SQLite database.</p>
</div>
<div class="viewer-tools">
<input id="viewer-search" type="search" placeholder="Search messages" />
<label class="viewer-auto-refresh">
<input id="viewer-auto-refresh" type="checkbox" checked />
<span>Auto-refresh</span>
</label>
<button id="viewer-refresh-btn" class="button button-small" type="button">Refresh</button>
</div>
</header>
<section id="messages-list" class="messages-list">
<div id="scroll-sentinel" class="scroll-sentinel"></div>
</section>
</main>
</div>
<template id="viewer-channel-template">
<button class="viewer-channel-item">
<span class="viewer-channel-avatar"></span>
<span class="viewer-channel-copy">
<span class="viewer-channel-row">
<span class="viewer-channel-name"></span>
<span class="viewer-channel-time"></span>
</span>
<span class="viewer-channel-row">
<span class="viewer-channel-preview"></span>
<span class="viewer-channel-count"></span>
</span>
</span>
</button>
</template>
<template id="message-template">
<div class="chat-bubble-wrap">
<div class="chat-bubble">
<div class="chat-sender"></div>
<div class="chat-reply">
<div class="chat-reply-border"></div>
<div class="chat-reply-body">
<div class="chat-reply-author"></div>
<div class="chat-reply-text"></div>
</div>
</div>
<div class="chat-text"></div>
<div class="chat-media"></div>
<div class="chat-footer"></div>
</div>
</div>
</template>
<script src="/static/viewer.js"></script>
</body>
</html>
+528
View File
@@ -0,0 +1,528 @@
if ('scrollRestoration' in history) history.scrollRestoration = 'manual';
const viewerState = {
accountId: null,
accounts: [],
channels: [],
channelId: null,
oldestMessageId: null,
newestMessageId: null,
userId: null,
search: '',
loading: false,
autoRefreshTimer: null,
sentinelObserver: null,
};
async function api(path) {
const response = await fetch(path);
const data = await response.json();
if (!response.ok) {
throw new Error(data.error || 'Request failed');
}
return data;
}
function showToast(message, type = 'info') {
let root = document.getElementById('toast-root');
if (!root) {
root = document.createElement('div');
root.id = 'toast-root';
root.className = 'toast-root';
document.body.appendChild(root);
}
const toast = document.createElement('div');
toast.className = `toast toast-${type}`;
toast.textContent = message;
root.appendChild(toast);
window.setTimeout(() => {
toast.classList.add('toast-hide');
window.setTimeout(() => toast.remove(), 250);
}, 3600);
}
function formatDateHeader(dateStr) {
if (!dateStr) return '';
const d = new Date(dateStr.replace(' ', 'T'));
if (isNaN(d.getTime())) return '';
const now = new Date();
const diff = now - d;
const oneDay = 86400000;
if (diff < oneDay && d.getDate() === now.getDate()) return 'Today';
if (diff < 2 * oneDay && d.getDate() === now.getDate() - 1) return 'Yesterday';
return d.toLocaleDateString('en-US', {
month: 'long',
day: 'numeric',
year: d.getFullYear() !== now.getFullYear() ? 'numeric' : undefined,
});
}
function formatTime(dateStr) {
if (!dateStr) return '';
const d = new Date(dateStr.replace(' ', 'T'));
if (isNaN(d.getTime())) return '';
return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
}
function chatListTime(dateStr) {
if (!dateStr) return '';
const d = new Date(dateStr.replace(' ', 'T'));
if (isNaN(d.getTime())) return '';
const now = new Date();
if (d.toDateString() === now.toDateString()) {
return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
}
return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
}
function initials(value) {
const source = String(value || '?')
.replace(/^@/, '')
.trim();
const words = source.split(/\s+/).filter(Boolean);
if (words.length > 1) return (words[0][0] + words[1][0]).toUpperCase();
return source.slice(0, 2).toUpperCase();
}
function dateKey(dateStr) {
if (!dateStr) return '';
return dateStr.slice(0, 10);
}
function textNodeWithBreaks(text) {
const fragment = document.createDocumentFragment();
if (!text) return fragment;
const parts = text.split('\n');
parts.forEach((part, index) => {
if (index > 0) fragment.appendChild(document.createElement('br'));
fragment.appendChild(document.createTextNode(part));
});
return fragment;
}
function scrollToBottom() {
const list = document.getElementById('messages-list');
if (!list) return;
const go = () => {
list.scrollTop = list.scrollHeight;
};
go();
requestAnimationFrame(go);
setTimeout(go, 150);
}
function makeScrollSentinel() {
const sentinel = document.createElement('div');
sentinel.id = 'scroll-sentinel';
sentinel.className = 'scroll-sentinel';
return sentinel;
}
function resetMessageView(title = 'Select a channel', subtitle = 'Reading messages from the local SQLite database.') {
const root = document.getElementById('messages-list');
if (root) {
root.innerHTML = '';
root.appendChild(makeScrollSentinel());
}
document.getElementById('viewer-title').textContent = title;
document.getElementById('viewer-subtitle').textContent = subtitle;
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
}
function renderChannelList() {
const root = document.getElementById('viewer-channel-list');
if (!root) return;
root.innerHTML = '';
const template = document.getElementById('viewer-channel-template');
if (!template) return;
viewerState.channels.forEach((channel) => {
const node = template.content.firstElementChild.cloneNode(true);
const preview =
channel.last_message_preview ||
(channel.has_database ? 'No text in the last saved message' : 'No local database yet');
node.querySelector('.viewer-channel-avatar').textContent = initials(channel.name || channel.channel_id);
node.querySelector('.viewer-channel-name').textContent = channel.name;
node.querySelector('.viewer-channel-time').textContent = chatListTime(channel.last_date);
node.querySelector('.viewer-channel-preview').textContent = preview;
node.querySelector('.viewer-channel-count').textContent = String(channel.message_count || 0);
if (channel.channel_id === viewerState.channelId) {
node.classList.add('active');
}
node.addEventListener('click', () => {
document.querySelector('.viewer-sidebar')?.classList.remove('open');
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
loadChannel(channel.channel_id);
});
root.appendChild(node);
});
}
function messageEndpoint(channelId, before = '') {
const search = viewerState.search ? `&search=${encodeURIComponent(viewerState.search)}` : '';
if (viewerState.accountId) {
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
}
return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
}
function channelsEndpoint() {
if (viewerState.accountId) {
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels`;
}
return '/api/channels';
}
function renderAccountSelector() {
const sel = document.getElementById('viewer-account-select');
if (!sel) return;
sel.innerHTML = '';
const legacy = viewerState.accounts.length === 0;
if (legacy) {
const opt = document.createElement('option');
opt.value = '';
opt.textContent = 'Legacy';
sel.appendChild(opt);
} else {
viewerState.accounts.forEach((acc) => {
const opt = document.createElement('option');
opt.value = acc.id;
opt.textContent = acc.label || acc.id;
if (acc.id === viewerState.accountId) opt.selected = true;
sel.appendChild(opt);
});
}
}
async function switchViewerAccount(accountId) {
viewerState.accountId = accountId || null;
viewerState.search = '';
const searchInput = document.getElementById('viewer-search');
if (searchInput) searchInput.value = '';
viewerState.accounts.forEach((acc) => {
if (acc.id === accountId) {
const opts = document.getElementById('viewer-account-select')?.options;
if (opts) {
for (let i = 0; i < opts.length; i++) {
opts[i].selected = opts[i].value === accountId;
}
}
}
});
try {
const authData = viewerState.accountId
? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`)
: await api('/api/auth');
viewerState.userId = authData.user_id || null;
} catch (e) {
console.warn('Could not fetch user ID:', e);
}
const channels = await api(channelsEndpoint());
viewerState.channels = channels;
viewerState.channelId = channels[0]?.channel_id || null;
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
renderChannelList();
if (viewerState.channelId) {
await loadChannel(viewerState.channelId);
} else {
resetMessageView(
'No channels',
viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.',
);
}
}
async function loadViewerAccount(params) {
const requested = params.get('account');
try {
const payload = await api('/api/accounts');
viewerState.accounts = payload.accounts || [];
} catch {
viewerState.accounts = [];
}
const requestedExists = requested && viewerState.accounts.some((acc) => acc.id === requested);
viewerState.accountId = requestedExists ? requested : viewerState.accounts[0]?.id || null;
renderAccountSelector();
const sel = document.getElementById('viewer-account-select');
if (sel) {
sel.addEventListener('change', () => switchViewerAccount(sel.value));
}
try {
const authData = viewerState.accountId
? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`)
: await api('/api/auth');
viewerState.userId = authData.user_id || null;
} catch (e) {
console.warn('Could not fetch user ID:', e);
}
}
function buildMessageNode(message, root) {
const template = document.getElementById('message-template');
const wrap = template.content.firstElementChild.cloneNode(true);
const bubble = wrap.querySelector('.chat-bubble');
bubble.dataset.messageId = String(message.message_id);
const isOwn = viewerState.userId && message.sender_id === viewerState.userId;
if (isOwn) {
wrap.classList.add('chat-bubble-wrap--own');
}
const sender = bubble.querySelector('.chat-sender');
sender.textContent = message.sender_name || '';
const replyEl = bubble.querySelector('.chat-reply');
const replyMsg = message.reply_to_message;
if (message.reply_to) {
replyEl.classList.remove('hidden');
bubble.querySelector('.chat-reply-author').textContent = replyMsg?.sender_name || `#${message.reply_to}`;
bubble.querySelector('.chat-reply-text').textContent = replyMsg?.text || '(message not available)';
replyEl.addEventListener('click', () => {
const target = root.querySelector(`[data-message-id="${message.reply_to}"]`);
if (target) {
target.scrollIntoView({ behavior: 'smooth', block: 'center' });
target.classList.add('chat-bubble--highlight');
setTimeout(() => target.classList.remove('chat-bubble--highlight'), 2000);
}
});
} else {
replyEl.classList.add('hidden');
}
bubble.querySelector('.chat-text').appendChild(textNodeWithBreaks(message.text || ''));
const media = bubble.querySelector('.chat-media');
if (message.media_url && message.media_kind === 'image') {
const img = document.createElement('img');
img.src = message.media_url;
img.loading = 'lazy';
media.appendChild(img);
} else if (message.media_url && message.media_kind === 'video') {
const video = document.createElement('video');
video.src = message.media_url;
video.controls = true;
video.preload = 'metadata';
media.appendChild(video);
} else if (message.media_url) {
const link = document.createElement('a');
link.href = message.media_url;
link.target = '_blank';
link.rel = 'noreferrer';
link.textContent = 'Open file';
media.appendChild(link);
}
const footerParts = [];
if (message.date) footerParts.push(formatTime(message.date));
if (message.views) footerParts.push(`views: ${message.views}`);
if (message.forwards) footerParts.push(`forwards: ${message.forwards}`);
if (message.reactions) footerParts.push(`reactions: ${message.reactions}`);
bubble.querySelector('.chat-footer').textContent = footerParts.join(' | ');
return { wrap, dateKey: dateKey(message.date) };
}
function makeDateSep(text) {
const el = document.createElement('div');
el.className = 'chat-date-sep';
el.textContent = text;
return el;
}
function renderMessages(payload, append = false) {
const root = document.getElementById('messages-list');
if (!root) return;
let sentinel = document.getElementById('scroll-sentinel');
if (!sentinel) sentinel = makeScrollSentinel();
const channel = payload.channel;
document.getElementById('viewer-title').textContent = channel?.name || payload.channel_id;
document.getElementById('viewer-subtitle').textContent = channel
? `${payload.messages.length} shown${viewerState.search ? ` for "${viewerState.search}"` : ''}, ${channel.message_count} total, latest: ${channel.last_date || '-'}`
: 'No local database found for this channel yet.';
if (!append) {
root.innerHTML = '';
root.appendChild(sentinel);
}
let lastKey = null;
if (append) {
const firstExisting = root.querySelector('[data-message-id]');
if (firstExisting) {
const candidate = firstExisting.closest('.chat-bubble-wrap');
if (candidate && candidate._dateKey) lastKey = candidate._dateKey;
}
}
const items = payload.messages.map((m) => {
const { wrap, dateKey: dk } = buildMessageNode(m, root);
wrap._dateKey = dk;
return { wrap, dk };
});
if (append) {
for (let i = items.length - 1; i >= 0; i--) {
const { wrap, dk } = items[i];
const target = sentinel.nextSibling;
if (dk && dk !== lastKey) {
root.insertBefore(makeDateSep(formatDateHeader(payload.messages[i].date)), target);
lastKey = dk;
}
root.insertBefore(wrap, target);
}
} else {
for (let i = 0; i < items.length; i++) {
const { wrap, dk } = items[i];
if (dk && dk !== lastKey) {
root.appendChild(makeDateSep(formatDateHeader(payload.messages[i].date)));
lastKey = dk;
}
root.appendChild(wrap);
}
}
if (payload.messages.length) {
viewerState.oldestMessageId = payload.messages[0].message_id;
viewerState.newestMessageId = payload.messages[payload.messages.length - 1].message_id;
} else if (!append) {
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
const empty = document.createElement('div');
empty.className = 'viewer-empty-state';
empty.textContent = viewerState.search
? `No messages found for "${viewerState.search}".`
: 'No saved messages in this channel yet.';
root.appendChild(empty);
}
if (!append) {
requestAnimationFrame(() => scrollToBottom());
}
}
async function loadChannel(channelId, append = false) {
if (viewerState.loading) return;
viewerState.loading = true;
viewerState.channelId = channelId;
renderChannelList();
const before = append && viewerState.oldestMessageId ? `&before=${viewerState.oldestMessageId}` : '';
try {
const payload = await api(messageEndpoint(channelId, before));
if (append) {
const list = document.getElementById('messages-list');
const prevScrollHeight = list.scrollHeight;
renderMessages(payload, true);
requestAnimationFrame(() => {
list.scrollTop = list.scrollHeight - prevScrollHeight;
});
} else {
renderMessages(payload, false);
}
} catch (err) {
console.error('Failed to load messages:', err);
} finally {
viewerState.loading = false;
}
}
async function refreshCurrentChannel() {
if (!viewerState.channelId || viewerState.loading) return;
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
await loadChannel(viewerState.channelId);
}
function setupViewerTools() {
const searchInput = document.getElementById('viewer-search');
const refreshBtn = document.getElementById('viewer-refresh-btn');
const autoRefresh = document.getElementById('viewer-auto-refresh');
let searchTimer = null;
if (searchInput) {
searchInput.addEventListener('input', () => {
clearTimeout(searchTimer);
searchTimer = setTimeout(() => {
viewerState.search = searchInput.value.trim();
refreshCurrentChannel();
}, 250);
});
}
if (refreshBtn) {
refreshBtn.addEventListener('click', () => refreshCurrentChannel());
}
viewerState.autoRefreshTimer = window.setInterval(() => {
if (!autoRefresh || autoRefresh.checked) {
refreshCurrentChannel();
}
}, 30000);
}
function setupInfiniteScroll() {
const sentinel = document.getElementById('scroll-sentinel');
if (!sentinel) return;
if (viewerState.sentinelObserver) {
viewerState.sentinelObserver.disconnect();
}
viewerState.sentinelObserver = new IntersectionObserver(
(entries) => {
for (const entry of entries) {
if (entry.isIntersecting && viewerState.channelId && viewerState.oldestMessageId && !viewerState.loading) {
loadChannel(viewerState.channelId, true);
}
}
},
{ root: document.getElementById('messages-list'), threshold: 0.1 },
);
viewerState.sentinelObserver.observe(sentinel);
}
async function main() {
const params = new URLSearchParams(window.location.search);
await loadViewerAccount(params);
viewerState.channels = await api(channelsEndpoint());
const requestedChannel = params.get('channel');
const requestedChannelExists =
requestedChannel && viewerState.channels.some((channel) => channel.channel_id === requestedChannel);
viewerState.channelId = requestedChannelExists ? requestedChannel : viewerState.channels[0]?.channel_id || null;
renderChannelList();
if (viewerState.channelId) {
await loadChannel(viewerState.channelId);
} else {
resetMessageView(
'No channels',
viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.',
);
}
setupInfiniteScroll();
setupViewerTools();
const toggle = document.getElementById('sidebar-toggle');
const sidebar = document.querySelector('.viewer-sidebar');
if (toggle && sidebar) {
toggle.addEventListener('click', () => {
sidebar.classList.toggle('open');
});
sidebar.addEventListener('click', (e) => {
if (e.target === sidebar) {
sidebar.classList.remove('open');
}
});
}
}
main().catch((error) => {
console.error(error);
showToast(error.message, 'error');
});
+2840
View File
File diff suppressed because it is too large Load Diff