Compare commits
62 Commits
stable_cli
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 70ac1ce73e | |||
| 81e69865ce | |||
| 418c4d6192 | |||
| 5956ea36a8 | |||
| 93c8d00af6 | |||
| 08f8196bf9 | |||
| b963cf07df | |||
| 849bf97c97 | |||
| 145e7de7f4 | |||
| 27296caf69 | |||
| a11ed6c5ba | |||
| 56f15e0ba5 | |||
| 26b6344f7a | |||
| 6bc8d6ed36 | |||
| 01a0d9c6d4 | |||
| d3beeb6b2d | |||
| 24d4898865 | |||
| a74b8bfd14 | |||
| a2c5ca7f2b | |||
| 715cb448b7 | |||
| ec4805f027 | |||
| 5b27852b97 | |||
| c5d0a7c7e5 | |||
| 23bb370836 | |||
| ff1ca13a03 | |||
| 7aa5c84e63 | |||
| 881e7ddfca | |||
| 59cdad220a | |||
| 9b72c3128e | |||
| 3eb6bdf534 | |||
| f0d207bdea | |||
| 1bd385e054 | |||
| 93f2b94dd2 | |||
| eb69a89d91 | |||
| 62a6a492e0 | |||
| bb4e28a710 | |||
| 220d04b3a6 | |||
| 6e3966aeb5 | |||
| f5ef74bda6 | |||
| ac5beb831b | |||
| df644219de | |||
| 7af597b800 | |||
| c5361f2271 | |||
| 1dfe41950e | |||
| 856d04bd16 | |||
| d49fe28f08 | |||
| a78a708757 | |||
| 6851922a9e | |||
| ff000e88d4 | |||
| cc99f752b3 | |||
| be02eed378 | |||
| 5b8a95d73d | |||
| cf674b4943 | |||
| 612fa28ec6 | |||
| 47ef106ee1 | |||
| 46922a887f | |||
| 3565e31885 | |||
| 9094a52ccb | |||
| db3641ddf9 | |||
| 19e173212c | |||
| f57e1d63eb | |||
| 535b8195ef |
+215
@@ -0,0 +1,215 @@
|
|||||||
|
data/
|
||||||
|
.codex
|
||||||
|
.python-version
|
||||||
|
uv.lock
|
||||||
|
.uv-cache/
|
||||||
|
session/
|
||||||
|
session.session
|
||||||
|
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[codz]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py.cover
|
||||||
|
*.lcov
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
# Pipfile.lock
|
||||||
|
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# uv.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
# poetry.lock
|
||||||
|
# poetry.toml
|
||||||
|
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||||
|
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||||
|
# pdm.lock
|
||||||
|
# pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
|
||||||
|
# pixi
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||||
|
# pixi.lock
|
||||||
|
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||||
|
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||||
|
.pixi/*
|
||||||
|
!.pixi/config.toml
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule*
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# Redis
|
||||||
|
*.rdb
|
||||||
|
*.aof
|
||||||
|
*.pid
|
||||||
|
|
||||||
|
# RabbitMQ
|
||||||
|
mnesia/
|
||||||
|
rabbitmq/
|
||||||
|
rabbitmq-data/
|
||||||
|
|
||||||
|
# ActiveMQ
|
||||||
|
activemq-data/
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
# .idea/
|
||||||
|
|
||||||
|
# Abstra
|
||||||
|
# Abstra is an AI-powered process automation framework.
|
||||||
|
# Ignore directories containing user credentials, local state, and settings.
|
||||||
|
# Learn more at https://abstra.io/docs
|
||||||
|
.abstra/
|
||||||
|
|
||||||
|
# Visual Studio Code
|
||||||
|
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||||
|
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||||
|
# you could uncomment the following to ignore the entire vscode folder
|
||||||
|
# .vscode/
|
||||||
|
# Temporary file for partial code execution
|
||||||
|
tempCodeRunnerFile.py
|
||||||
|
|
||||||
|
# Ruff stuff:
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
|
||||||
|
# Marimo
|
||||||
|
marimo/_static/
|
||||||
|
marimo/_lsp/
|
||||||
|
__marimo__/
|
||||||
|
|
||||||
|
# Streamlit
|
||||||
|
.streamlit/secrets.toml
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
root = true
|
||||||
|
|
||||||
|
[*]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 2
|
||||||
|
end_of_line = lf
|
||||||
|
charset = utf-8
|
||||||
|
trim_trailing_whitespace = true
|
||||||
|
insert_final_newline = true
|
||||||
|
|
||||||
|
[*.{yml,yaml}]
|
||||||
|
indent_size = 2
|
||||||
|
|
||||||
|
[*.{json,jsonc}]
|
||||||
|
indent_size = 2
|
||||||
|
|
||||||
|
[*.md]
|
||||||
|
trim_trailing_whitespace = false
|
||||||
|
|
||||||
|
[*.py]
|
||||||
|
indent_size = 4
|
||||||
|
|
||||||
|
[{Makefile,makefile}]
|
||||||
|
indent_style = tab
|
||||||
@@ -0,0 +1,171 @@
|
|||||||
|
name: ci
|
||||||
|
|
||||||
|
'on':
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- '**'
|
||||||
|
pull_request:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Shared image coordinates for the publish job.
|
||||||
|
REGISTRY: gcr.forust.xyz
|
||||||
|
IMAGE_NAME: forust/telegram-scraper
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint-prettier:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Check formatting with Prettier
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
mapfile -t prettier_files < <(
|
||||||
|
git ls-files \
|
||||||
|
| grep -E '\.(md|json|ya?ml|html|css)$' \
|
||||||
|
| grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ "${#prettier_files[@]}" -eq 0 ]; then
|
||||||
|
echo "No Prettier-managed files found."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
node:22-alpine \
|
||||||
|
sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
|
||||||
|
|
||||||
|
lint-ruff:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Lint Python with Ruff
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
ghcr.io/astral-sh/ruff:latest \
|
||||||
|
check .
|
||||||
|
|
||||||
|
lint-yaml:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Lint YAML syntax
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
cytopia/yamllint:latest \
|
||||||
|
-c .yamllint .
|
||||||
|
|
||||||
|
lint-dockerfiles:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Lint Dockerfiles
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
mapfile -t dockerfiles < <(
|
||||||
|
git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ "${#dockerfiles[@]}" -eq 0 ]; then
|
||||||
|
echo "No Dockerfiles found."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
--entrypoint hadolint \
|
||||||
|
hadolint/hadolint:latest-debian \
|
||||||
|
-c .hadolint.yaml "${dockerfiles[@]}"
|
||||||
|
|
||||||
|
validate:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Validate Kubernetes manifests
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
mapfile -t manifests < <(
|
||||||
|
git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
|
||||||
|
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ "${#manifests[@]}" -eq 0 ]; then
|
||||||
|
echo "No Kubernetes manifests found."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
ghcr.io/yannh/kubeconform:latest \
|
||||||
|
-strict \
|
||||||
|
-ignore-missing-schemas \
|
||||||
|
-summary \
|
||||||
|
"${manifests[@]}"
|
||||||
|
|
||||||
|
publish:
|
||||||
|
needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
|
||||||
|
if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Read project version
|
||||||
|
id: version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')"
|
||||||
|
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Log in to registry
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
|
||||||
|
-u "${{ secrets.REGISTRY_USERNAME }}" \
|
||||||
|
--password-stdin
|
||||||
|
|
||||||
|
- name: Build and push image
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
image="${REGISTRY}/${IMAGE_NAME}"
|
||||||
|
tags=("latest" "${{ steps.version.outputs.version }}")
|
||||||
|
|
||||||
|
case "${GITHUB_REF_NAME}" in
|
||||||
|
main)
|
||||||
|
tags+=("main" "prod")
|
||||||
|
;;
|
||||||
|
dev)
|
||||||
|
tags+=("dev")
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
build_args=()
|
||||||
|
for tag in "${tags[@]}"; do
|
||||||
|
build_args+=(-t "${image}:${tag}")
|
||||||
|
done
|
||||||
|
|
||||||
|
docker build "${build_args[@]}" .
|
||||||
|
|
||||||
|
for tag in "${tags[@]}"; do
|
||||||
|
docker push "${image}:${tag}"
|
||||||
|
done
|
||||||
@@ -1 +0,0 @@
|
|||||||
ko_fi: unnohwn
|
|
||||||
@@ -0,0 +1,171 @@
|
|||||||
|
name: ci
|
||||||
|
|
||||||
|
'on':
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- '**'
|
||||||
|
pull_request:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Shared image coordinates for the publish job.
|
||||||
|
REGISTRY: gcr.forust.xyz
|
||||||
|
IMAGE_NAME: forust/telegram-scraper
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint-prettier:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Check formatting with Prettier
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
mapfile -t prettier_files < <(
|
||||||
|
git ls-files \
|
||||||
|
| grep -E '\.(md|json|ya?ml|html|css)$' \
|
||||||
|
| grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ "${#prettier_files[@]}" -eq 0 ]; then
|
||||||
|
echo "No Prettier-managed files found."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
node:22-alpine \
|
||||||
|
sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
|
||||||
|
|
||||||
|
lint-ruff:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Lint Python with Ruff
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
ghcr.io/astral-sh/ruff:latest \
|
||||||
|
check .
|
||||||
|
|
||||||
|
lint-yaml:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Lint YAML syntax
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
cytopia/yamllint:latest \
|
||||||
|
-c .yamllint .
|
||||||
|
|
||||||
|
lint-dockerfiles:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Lint Dockerfiles
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
mapfile -t dockerfiles < <(
|
||||||
|
git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ "${#dockerfiles[@]}" -eq 0 ]; then
|
||||||
|
echo "No Dockerfiles found."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
--entrypoint hadolint \
|
||||||
|
hadolint/hadolint:latest-debian \
|
||||||
|
-c .hadolint.yaml "${dockerfiles[@]}"
|
||||||
|
|
||||||
|
validate:
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Validate Kubernetes manifests
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
mapfile -t manifests < <(
|
||||||
|
git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
|
||||||
|
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ "${#manifests[@]}" -eq 0 ]; then
|
||||||
|
echo "No Kubernetes manifests found."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker run --rm \
|
||||||
|
-v "$PWD:/work" \
|
||||||
|
-w /work \
|
||||||
|
ghcr.io/yannh/kubeconform:latest \
|
||||||
|
-strict \
|
||||||
|
-ignore-missing-schemas \
|
||||||
|
-summary \
|
||||||
|
"${manifests[@]}"
|
||||||
|
|
||||||
|
publish:
|
||||||
|
needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
|
||||||
|
if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
|
||||||
|
runs-on: [self-hosted, linux, arch, homelab]
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Read project version
|
||||||
|
id: version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
version="$(python3 -c 'from pathlib import Path; import tomllib; print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])')"
|
||||||
|
echo "version=$version" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Log in to registry
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
|
||||||
|
-u "${{ secrets.REGISTRY_USERNAME }}" \
|
||||||
|
--password-stdin
|
||||||
|
|
||||||
|
- name: Build and push image
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
image="${REGISTRY}/${IMAGE_NAME}"
|
||||||
|
tags=("latest" "${{ steps.version.outputs.version }}")
|
||||||
|
|
||||||
|
case "${GITHUB_REF_NAME}" in
|
||||||
|
main)
|
||||||
|
tags+=("main" "prod")
|
||||||
|
;;
|
||||||
|
dev)
|
||||||
|
tags+=("dev")
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
build_args=()
|
||||||
|
for tag in "${tags[@]}"; do
|
||||||
|
build_args+=(-t "${image}:${tag}")
|
||||||
|
done
|
||||||
|
|
||||||
|
docker build "${build_args[@]}" .
|
||||||
|
|
||||||
|
for tag in "${tags[@]}"; do
|
||||||
|
docker push "${image}:${tag}"
|
||||||
|
done
|
||||||
+228
@@ -0,0 +1,228 @@
|
|||||||
|
data/
|
||||||
|
.codex
|
||||||
|
.python-version
|
||||||
|
uv.lock
|
||||||
|
.uv-cache/
|
||||||
|
session/
|
||||||
|
session.session
|
||||||
|
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[codz]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py.cover
|
||||||
|
*.lcov
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
# Pipfile.lock
|
||||||
|
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# uv.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
# poetry.lock
|
||||||
|
# poetry.toml
|
||||||
|
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||||
|
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||||
|
# pdm.lock
|
||||||
|
# pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
|
||||||
|
# pixi
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||||
|
# pixi.lock
|
||||||
|
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||||
|
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||||
|
.pixi/*
|
||||||
|
!.pixi/config.toml
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule*
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# Redis
|
||||||
|
*.rdb
|
||||||
|
*.aof
|
||||||
|
*.pid
|
||||||
|
|
||||||
|
# RabbitMQ
|
||||||
|
mnesia/
|
||||||
|
rabbitmq/
|
||||||
|
rabbitmq-data/
|
||||||
|
|
||||||
|
# ActiveMQ
|
||||||
|
activemq-data/
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.envrc
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
# .idea/
|
||||||
|
|
||||||
|
# Abstra
|
||||||
|
# Abstra is an AI-powered process automation framework.
|
||||||
|
# Ignore directories containing user credentials, local state, and settings.
|
||||||
|
# Learn more at https://abstra.io/docs
|
||||||
|
.abstra/
|
||||||
|
|
||||||
|
# Visual Studio Code
|
||||||
|
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||||
|
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||||
|
# you could uncomment the following to ignore the entire vscode folder
|
||||||
|
# .vscode/
|
||||||
|
# Temporary file for partial code execution
|
||||||
|
tempCodeRunnerFile.py
|
||||||
|
|
||||||
|
# Ruff stuff:
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
|
||||||
|
# Marimo
|
||||||
|
marimo/_static/
|
||||||
|
marimo/_lsp/
|
||||||
|
__marimo__/
|
||||||
|
|
||||||
|
# Streamlit
|
||||||
|
.streamlit/secrets.toml
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
ignored:
|
||||||
|
- DL3008
|
||||||
|
- DL3042
|
||||||
|
- DL3018
|
||||||
|
- DL3059
|
||||||
|
trustedRegistries:
|
||||||
|
- docker.io
|
||||||
|
- ghcr.io
|
||||||
|
- quay.io
|
||||||
|
- gcr.forust.xyz
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"default": true,
|
||||||
|
"MD013": false,
|
||||||
|
"MD024": false,
|
||||||
|
"MD033": false,
|
||||||
|
"MD041": false,
|
||||||
|
"MD046": false
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
bracketSameLine: true
|
||||||
|
htmlWhitespaceSensitivity: css
|
||||||
|
printWidth: 120
|
||||||
|
tabWidth: 2
|
||||||
|
singleQuote: true
|
||||||
|
trailingComma: all
|
||||||
|
proseWrap: preserve
|
||||||
|
endOfLine: lf
|
||||||
Vendored
+34
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"[yaml]": {
|
||||||
|
"editor.tabSize": 2,
|
||||||
|
"editor.insertSpaces": true,
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"editor.defaultFormatter": "redhat.vscode-yaml"
|
||||||
|
},
|
||||||
|
"[python]": {
|
||||||
|
"editor.tabSize": 4,
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"editor.defaultFormatter": "charliermarsh.ruff"
|
||||||
|
},
|
||||||
|
"[dockerfile]": {
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"editor.defaultFormatter": "exiasr.hadolint"
|
||||||
|
},
|
||||||
|
"[markdown]": {
|
||||||
|
"editor.wordWrap": "wordWrapColumn",
|
||||||
|
"editor.wordWrapColumn": 120
|
||||||
|
},
|
||||||
|
"yaml.schemas": {
|
||||||
|
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
|
||||||
|
},
|
||||||
|
"yaml.format.enable": true,
|
||||||
|
"yaml.validate": true,
|
||||||
|
"yaml.completion": true,
|
||||||
|
"prettier.bracketSameLine": true,
|
||||||
|
"prettier.htmlWhitespaceSensitivity": "css",
|
||||||
|
"prettier.printWidth": 120,
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"files.autoSave": "onFocusChange",
|
||||||
|
"editor.wordWrapColumn": 120,
|
||||||
|
"editor.wordWrap": "wordWrapColumn"
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
extends: default
|
||||||
|
|
||||||
|
rules:
|
||||||
|
comments:
|
||||||
|
min-spaces-from-content: 1
|
||||||
|
comments-indentation: false
|
||||||
|
document-start: disable
|
||||||
|
line-length: disable
|
||||||
|
braces:
|
||||||
|
min-spaces-inside: 0
|
||||||
|
max-spaces-inside: 1
|
||||||
|
brackets:
|
||||||
|
min-spaces-inside: 0
|
||||||
|
max-spaces-inside: 1
|
||||||
|
indentation:
|
||||||
|
spaces: 2
|
||||||
|
indent-sequences: consistent
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"tab_size": 2,
|
||||||
|
"soft_wrap": "prefer_line",
|
||||||
|
"preferred_line_length": 120,
|
||||||
|
"format_on_save": "on",
|
||||||
|
"languages": {
|
||||||
|
"YAML": {
|
||||||
|
"tab_size": 2,
|
||||||
|
"hard_tabs": false,
|
||||||
|
"format_on_save": "on",
|
||||||
|
"formatter": {
|
||||||
|
"language_server": { "name": "yaml-language-server" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Python": {
|
||||||
|
"tab_size": 4,
|
||||||
|
"format_on_save": "on",
|
||||||
|
"formatter": {
|
||||||
|
"language_server": { "name": "ruff" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"lsp": {
|
||||||
|
"yaml-language-server": {
|
||||||
|
"settings": {
|
||||||
|
"yaml": {
|
||||||
|
"schemas": {
|
||||||
|
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
|
||||||
|
},
|
||||||
|
"validate": true,
|
||||||
|
"completion": true,
|
||||||
|
"format": {
|
||||||
|
"enable": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+18
@@ -0,0 +1,18 @@
|
|||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
gcc \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
VOLUME ["/app/data"]
|
||||||
|
VOLUME ["/app/session"]
|
||||||
|
|
||||||
|
CMD ["python", "main.py"]
|
||||||
@@ -1,207 +1,233 @@
|
|||||||
# Telegram Channel Scraper 📱
|
# Telegram Scraper
|
||||||
|
|
||||||
A powerful Python script that allows you to scrape messages and media from Telegram channels using the Telethon library. Features include real-time continuous scraping, media downloading, and data export capabilities.
|
Telegram scraper on top of Telethon with:
|
||||||
|
|
||||||
```
|
- CLI workflow for scraping, exports, media recovery, and forwarding
|
||||||
___________________ _________
|
- lightweight multi-account Web UI served by Python
|
||||||
\__ ___/ _____/ / _____/
|
- SQLite storage per tracked chat/channel
|
||||||
| | / \ ___ \_____ \
|
- Docker/Compose / Kubernetes setup for "run it and leave it on the server"
|
||||||
| | \ \_\ \/ \
|
|
||||||
|____| \______ /_______ /
|
## What It Does
|
||||||
\/ \/
|
|
||||||
|
- Scrapes messages with metadata: views, forwards, reactions, post author
|
||||||
|
- Downloads media into per-channel folders
|
||||||
|
- Stores messages in SQLite
|
||||||
|
- Exports to CSV and JSON
|
||||||
|
- Supports continuous scraping
|
||||||
|
- Supports forwarding rules
|
||||||
|
- Multi-account support — multiple Telegram sessions side by side
|
||||||
|
- Lets you browse tracked channels and local exports in a web panel with account tabs
|
||||||
|
|
||||||
|
## Project Layout
|
||||||
|
|
||||||
|
```text
|
||||||
|
.
|
||||||
|
├── main.py # starts the web server
|
||||||
|
├── webui_server.py # HTTP server + REST API
|
||||||
|
├── webui/ # plain HTML/CSS/JS frontend
|
||||||
|
│ ├── index.html / app.js # main dashboard
|
||||||
|
│ ├── viewer.html / viewer.js # message viewer with account selector
|
||||||
|
│ └── swagger.html / swagger.js # API docs
|
||||||
|
├── telegram_scraper_with_forwarding.py
|
||||||
|
├── app_state.py # multi-account state management
|
||||||
|
├── scraper_jobs.py # async job runner
|
||||||
|
├── health.py # health check endpoints
|
||||||
|
├── data/
|
||||||
|
│ ├── state.json # global state (accounts list)
|
||||||
|
│ └── accounts/<id>/ # per-account data
|
||||||
|
│ ├── state.json # per-account settings & credentials
|
||||||
|
│ └── <channel_id>/ # SQLite DB + media
|
||||||
|
└── session/
|
||||||
|
├── <account_id>.session # per-account Telethon session
|
||||||
|
└── ... # (WAL journal mode for concurrency)
|
||||||
```
|
```
|
||||||
|
|
||||||
## What's New in v3.1 🎉
|
## Requirements
|
||||||
|
|
||||||
**Enhanced Message Data:**
|
- Python 3.11+
|
||||||
- **Message statistics** - Captures views, forwards, and post_author for each message
|
|
||||||
- **Reactions support** - Records all emoji reactions with counts (e.g., "😀 12 👍 3")
|
|
||||||
- **Automatic database migration** - Seamlessly adds new columns to existing databases
|
|
||||||
- **Richer exports** - All new data included in CSV/JSON exports
|
|
||||||
|
|
||||||
**Improved Channel Management:**
|
|
||||||
- **Channel names displayed** - Shows channel names alongside IDs everywhere
|
|
||||||
- **Smart filtering** - List option now only shows Channels and Groups (no private chats)
|
|
||||||
- **channels_list.csv export** - Automatically saves channel list with names, IDs, usernames, and types
|
|
||||||
- **"all" selection** - Quickly add all listed channels at once
|
|
||||||
- **Better export naming** - Files now named as `ID_username.csv` and `ID_username.json`
|
|
||||||
|
|
||||||
**Bug Fixes:**
|
|
||||||
- **Fixed channel ID parsing** - Resolved "invalid literal for int()" error in fix missing media
|
|
||||||
- **Better entity resolution** - Handles both numeric IDs and channel usernames
|
|
||||||
- **Improved error messages** - Shows channel names with IDs for clearer debugging
|
|
||||||
|
|
||||||
## Features 🚀
|
|
||||||
|
|
||||||
- **QR Code & Phone Authentication** - Choose your preferred login method
|
|
||||||
- Scrape messages with full metadata (views, forwards, reactions, post author)
|
|
||||||
- Download media files with parallel processing and unique naming
|
|
||||||
- Real-time continuous scraping
|
|
||||||
- Export data to JSON and CSV formats with enhanced metadata
|
|
||||||
- SQLite database storage with automatic schema migration
|
|
||||||
- Resume capability (saves progress)
|
|
||||||
- Interactive menu with channel names and numbered selection
|
|
||||||
- Smart channel filtering (only shows channels/groups)
|
|
||||||
- Progress tracking with visual progress bars
|
|
||||||
- Automatic channels list export to CSV
|
|
||||||
|
|
||||||
## Prerequisites 📋
|
|
||||||
|
|
||||||
Before running the script, you'll need:
|
|
||||||
|
|
||||||
- Python 3.7 or higher
|
|
||||||
- Telegram account
|
- Telegram account
|
||||||
- API credentials from Telegram
|
- `api_id` and `api_hash` from https://my.telegram.org
|
||||||
|
|
||||||
### Required Python packages
|
## Telegram API Credentials
|
||||||
|
|
||||||
```
|
1. Go to https://my.telegram.org/auth
|
||||||
pip install -r requirements.txt
|
2. Sign in with your Telegram account
|
||||||
```
|
3. Open `API development tools`
|
||||||
|
4. Create an application
|
||||||
|
5. Save:
|
||||||
|
- `api_id`
|
||||||
|
- `api_hash`
|
||||||
|
|
||||||
## Getting Telegram API Credentials 🔑
|
## Install
|
||||||
|
|
||||||
1. Visit https://my.telegram.org/auth
|
### Local
|
||||||
2. Log in with your phone number
|
|
||||||
3. Click on "API development tools"
|
|
||||||
4. Fill in the form:
|
|
||||||
- App title: Your app name
|
|
||||||
- Short name: Your app short name
|
|
||||||
- Platform: Can be left as "Desktop"
|
|
||||||
- Description: Brief description of your app
|
|
||||||
5. Click "Create application"
|
|
||||||
6. You'll receive:
|
|
||||||
- `api_id`: A number
|
|
||||||
- `api_hash`: A string of letters and numbers
|
|
||||||
|
|
||||||
Keep these credentials safe, you'll need them to run the script!
|
|
||||||
|
|
||||||
## Setup and Running 🔧
|
|
||||||
|
|
||||||
1. Clone the repository:
|
|
||||||
```bash
|
|
||||||
git clone https://github.com/unnohwn/telegram-scraper.git
|
|
||||||
cd telegram-scraper
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Install requirements:
|
|
||||||
```bash
|
```bash
|
||||||
pip install -r requirements.txt
|
pip install -r requirements.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
3. Run the script:
|
Or with `uv`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python telegram-scraper.py
|
uv sync
|
||||||
```
|
```
|
||||||
|
|
||||||
4. On first run, you'll be prompted to enter:
|
## Smoke Test
|
||||||
- Your API ID (from my.telegram.org)
|
|
||||||
- Your API Hash (from my.telegram.org)
|
|
||||||
- **Choose authentication method:**
|
|
||||||
- **QR Code** (Recommended) - Scan with your phone (no phone number needed)
|
|
||||||
- **Phone Number** - Traditional SMS verification
|
|
||||||
|
|
||||||
## Usage 📝
|
A smoke test is a quick "does it start and answer basic requests?" check. It does
|
||||||
|
not replace full tests, but it catches broken imports, routes, and JSON responses.
|
||||||
|
|
||||||
The script provides a clean interactive menu:
|
```bash
|
||||||
|
python scripts/smoke_test.py
|
||||||
```
|
|
||||||
========================================
|
|
||||||
TELEGRAM SCRAPER
|
|
||||||
========================================
|
|
||||||
[S] Scrape channels
|
|
||||||
[C] Continuous scraping
|
|
||||||
[M] Media scraping: ON
|
|
||||||
[L] List & add channels
|
|
||||||
[R] Remove channels
|
|
||||||
[E] Export data
|
|
||||||
[T] Rescrape media
|
|
||||||
[Q] Quit
|
|
||||||
========================================
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Channel Selection Made Easy 🔢
|
The smoke test starts the web server on `127.0.0.1:18080` and disables automatic
|
||||||
|
continuous scraping for that process.
|
||||||
|
|
||||||
Instead of typing long channel IDs, use numbers:
|
## Running
|
||||||
|
|
||||||
**Adding Channels:**
|
### Web UI
|
||||||
```
|
|
||||||
[1] Tech News (ID: -1002116176890, Type: Channel, Username: @technews)
|
|
||||||
[2] Python Dev (ID: -1001597139842, Type: Group, Username: @pythondev)
|
|
||||||
[3] Daily Updates (ID: -1002274713954, Type: Channel, Username: @dailyupdates)
|
|
||||||
|
|
||||||
Enter: 1,3 (adds channels 1 and 3)
|
Starts on `0.0.0.0:8080` by default:
|
||||||
Or: all (adds all listed channels)
|
|
||||||
|
```bash
|
||||||
|
python main.py
|
||||||
```
|
```
|
||||||
|
|
||||||
**Viewing Your Channels:**
|
Or:
|
||||||
```
|
|
||||||
[1] Tech News (ID: -1002116176890), Last Message ID: 5234, Messages: 12450
|
```bash
|
||||||
[2] Python Dev (ID: -1001597139842), Last Message ID: 8192, Messages: 45782
|
uv run python main.py
|
||||||
```
|
```
|
||||||
|
|
||||||
**Scraping Channels:**
|
Open:
|
||||||
- Single: `1`
|
|
||||||
- Multiple: `1,3,5`
|
|
||||||
- All: `all`
|
|
||||||
- Mix formats: `1,-1001597139842,3`
|
|
||||||
|
|
||||||
## Data Storage 💾
|
```text
|
||||||
|
http://<server-ip>:8080
|
||||||
|
```
|
||||||
|
|
||||||
### Database Structure
|
The current web panel includes:
|
||||||
|
|
||||||
Data is stored in SQLite databases, one per channel:
|
- multi-account support with account tabs
|
||||||
- Location: `./channelname/channelname.db`
|
- tracked channels overview per account
|
||||||
- Optimized with indexes for fast queries
|
- background job queue
|
||||||
- WAL mode for better performance
|
- scrape/export/media actions per account
|
||||||
- Schema includes: message_id, date, sender info, message text, media info, reply_to, post_author, views, forwards, reactions
|
- shared `scrape_media` toggle
|
||||||
- Automatic migration adds new columns to existing databases
|
- local message viewer with account selector
|
||||||
|
- API docs at `/swagger` and `/openapi.json`
|
||||||
|
- health checks at `/health` and `/health/continuous`
|
||||||
|
|
||||||
### Media Storage 📁
|
### CLI
|
||||||
|
|
||||||
Media files are stored with unique naming:
|
If you still want the old interactive mode:
|
||||||
- Location: `./channelname/media/`
|
|
||||||
- Format: `{message_id}-{unique_id}-{original_name}.ext`
|
|
||||||
- **No more file overwrites** - Each file gets a unique name
|
|
||||||
|
|
||||||
### Exported Data 📊
|
```bash
|
||||||
|
python telegram_scraper_with_forwarding.py
|
||||||
|
```
|
||||||
|
|
||||||
Export formats:
|
## Docker / Kubernetes
|
||||||
1. **CSV**: `./channelname/channelid_username.csv`
|
|
||||||
2. **JSON**: `./channelname/channelid_username.json`
|
|
||||||
3. **Channel List**: `./channels_list.csv` (automatically created when using [L] option)
|
|
||||||
|
|
||||||
All exports include complete message metadata: views, forwards, reactions, and post author information.
|
### Docker Compose
|
||||||
|
|
||||||
## Performance Features ⚙️
|
```bash
|
||||||
|
docker compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
- **5 concurrent downloads** for faster media processing
|
### Kubernetes
|
||||||
- **Batch database operations** for optimal speed
|
|
||||||
- **Progress bars** with real-time feedback
|
|
||||||
- **Resume capability** - Continue where you left off
|
|
||||||
- **Memory-efficient** exports for large datasets
|
|
||||||
|
|
||||||
## Error Handling 🛠️
|
```bash
|
||||||
|
kubectl apply -f k8s/telegram-scraper.yaml
|
||||||
|
```
|
||||||
|
|
||||||
- Automatic retry with exponential backoff
|
Then open:
|
||||||
- Rate limit compliance
|
|
||||||
- Network error recovery
|
|
||||||
- State preservation during interruptions
|
|
||||||
|
|
||||||
## Limitations ⚠️
|
```text
|
||||||
|
http://<server-ip>:7887
|
||||||
|
```
|
||||||
|
|
||||||
- Respects Telegram's rate limits
|
### Volumes (Docker)
|
||||||
- Can only access public channels or channels you're a member of
|
|
||||||
- Media download size limits apply as per Telegram's restrictions
|
|
||||||
|
|
||||||
## License 📄
|
- `./data:/app/data` for databases, exports, and `state.json`
|
||||||
|
- `session:/app/session` for Telethon sessions
|
||||||
|
|
||||||
This project is licensed under the MIT License - see the LICENSE file for details.
|
If you already logged in before with the same compose volume and did not remove it, the session should be reused.
|
||||||
|
|
||||||
## Disclaimer ⚖️
|
## Multi-Account
|
||||||
|
|
||||||
This tool is for educational purposes only. Make sure to:
|
The app supports multiple Telegram accounts side by side. Each account has:
|
||||||
- Respect Telegram's Terms of Service
|
|
||||||
- Obtain necessary permissions before scraping
|
- its own `state.json` under `data/accounts/<id>/`
|
||||||
- Use responsibly and ethically
|
- its own Telethon session file `session/<id>.session`
|
||||||
- Comply with data protection regulations
|
- its own channel databases and media
|
||||||
|
|
||||||
|
Use the Web UI account tabs to switch between accounts, or create new ones in Settings.
|
||||||
|
|
||||||
|
Legacy single-account data is automatically migrated to the `default` account on first startup.
|
||||||
|
|
||||||
|
## Shared State
|
||||||
|
|
||||||
|
The Web UI and CLI share the same files:
|
||||||
|
|
||||||
|
- `data/state.json` (global — accounts list)
|
||||||
|
- `data/accounts/<id>/state.json` (per-account credentials and settings)
|
||||||
|
- `session/<id>.session` (per-account Telethon session, WAL journal mode)
|
||||||
|
|
||||||
|
That means:
|
||||||
|
|
||||||
|
- credentials persist in state files
|
||||||
|
- both entry points use the same tracked channels per account
|
||||||
|
- both entry points can reuse the same Telegram login sessions
|
||||||
|
|
||||||
|
## Data Storage
|
||||||
|
|
||||||
|
### SQLite
|
||||||
|
|
||||||
|
Each tracked channel per account gets its own database:
|
||||||
|
|
||||||
|
```text
|
||||||
|
data/accounts/<account_id>/<channel_id>/<channel_id>.db
|
||||||
|
```
|
||||||
|
|
||||||
|
Main fields include:
|
||||||
|
|
||||||
|
- `message_id`
|
||||||
|
- `date`
|
||||||
|
- `sender_id`
|
||||||
|
- `first_name`
|
||||||
|
- `last_name`
|
||||||
|
- `username`
|
||||||
|
- `message`
|
||||||
|
- `media_type`
|
||||||
|
- `media_path`
|
||||||
|
- `reply_to`
|
||||||
|
- `post_author`
|
||||||
|
- `views`
|
||||||
|
- `forwards`
|
||||||
|
- `reactions`
|
||||||
|
|
||||||
|
### Media
|
||||||
|
|
||||||
|
Media files are stored in:
|
||||||
|
|
||||||
|
```text
|
||||||
|
data/accounts/<account_id>/<channel_id>/media/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Exports
|
||||||
|
|
||||||
|
Generated into the same channel folder:
|
||||||
|
|
||||||
|
- `data/accounts/<account_id>/<channel_id>/<channel_id>_<username>.csv`
|
||||||
|
- `data/accounts/<account_id>/<channel_id>/<channel_id>_<username>.json`
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- The web server is intentionally simple: plain HTML/CSS/JS, no frontend framework
|
||||||
|
- The viewer reads local SQLite data and supports account switching via a dropdown
|
||||||
|
- Session SQLite files use WAL journal mode to prevent "database is locked" errors when concurrent scraping and jobs run
|
||||||
|
- The active account tab is persisted in `localStorage` across page reloads
|
||||||
|
- If dependencies are installed but the Telegram session is missing, the Web UI falls back to read-only status until login is available
|
||||||
|
|
||||||
|
## Disclaimer
|
||||||
|
|
||||||
|
Use responsibly and make sure your usage complies with Telegram rules, local law, and any privacy obligations.
|
||||||
|
|||||||
+313
@@ -0,0 +1,313 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from copy import deepcopy
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ── defaults ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
GLOBAL_DEFAULTS: Dict[str, Any] = {
|
||||||
|
"accounts": [],
|
||||||
|
"version": 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
ACCOUNT_DEFAULTS: Dict[str, Any] = {
|
||||||
|
"label": "",
|
||||||
|
"api_id": None,
|
||||||
|
"api_hash": None,
|
||||||
|
"channels": {},
|
||||||
|
"channel_names": {},
|
||||||
|
"scrape_media": True,
|
||||||
|
"forwarding_rules": [],
|
||||||
|
"continuous_scraping": {
|
||||||
|
"enabled": False,
|
||||||
|
"interval_minutes": 1,
|
||||||
|
"channels": [],
|
||||||
|
"run_all_tracked": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── StateStore (thread-safe JSON) ─────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class StateStore:
|
||||||
|
"""Thread-safe JSON state store with TTL cache."""
|
||||||
|
|
||||||
|
def __init__(self, path: Path, defaults: Optional[Dict[str, Any]] = None):
|
||||||
|
self.path = path
|
||||||
|
self.defaults = defaults or {}
|
||||||
|
self.lock = threading.RLock()
|
||||||
|
self._cache: Optional[Dict[str, Any]] = None
|
||||||
|
self._cache_time: float = 0
|
||||||
|
self._cache_ttl: float = 1.0
|
||||||
|
|
||||||
|
def load(self) -> Dict[str, Any]:
|
||||||
|
with self.lock:
|
||||||
|
now = time.time()
|
||||||
|
if self._cache is not None and (now - self._cache_time) < self._cache_ttl:
|
||||||
|
return dict(self._cache)
|
||||||
|
if not self.path.exists():
|
||||||
|
result = deepcopy(self.defaults)
|
||||||
|
self._cache = result
|
||||||
|
self._cache_time = now
|
||||||
|
return result
|
||||||
|
try:
|
||||||
|
with self.path.open("r", encoding="utf-8") as handle:
|
||||||
|
state: Dict[str, Any] = json.load(handle)
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
result = deepcopy(self.defaults)
|
||||||
|
self._cache = result
|
||||||
|
self._cache_time = now
|
||||||
|
return result
|
||||||
|
result = self._merge_defaults(state)
|
||||||
|
self._cache = result
|
||||||
|
self._cache_time = now
|
||||||
|
return result
|
||||||
|
|
||||||
|
def save(self, state: Dict[str, Any]) -> None:
|
||||||
|
with self.lock:
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
|
||||||
|
merged = self._merge_defaults(state)
|
||||||
|
with tmp_path.open("w", encoding="utf-8") as handle:
|
||||||
|
json.dump(merged, handle, ensure_ascii=False, indent=2)
|
||||||
|
handle.write("\n")
|
||||||
|
tmp_path.replace(self.path)
|
||||||
|
self._cache = None
|
||||||
|
|
||||||
|
def update(self, mutator: Callable[[Dict[str, Any]], None]) -> Dict[str, Any]:
|
||||||
|
with self.lock:
|
||||||
|
state = self.load()
|
||||||
|
mutator(state)
|
||||||
|
self.save(state)
|
||||||
|
return state
|
||||||
|
|
||||||
|
def continuous_config(self) -> Dict[str, Any]:
|
||||||
|
state = self.load()
|
||||||
|
return deepcopy(state.get("continuous_scraping") or ACCOUNT_DEFAULTS["continuous_scraping"])
|
||||||
|
|
||||||
|
def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
def mutate(state: Dict[str, Any]) -> None:
|
||||||
|
state["continuous_scraping"] = {
|
||||||
|
"enabled": bool(config.get("enabled", True)),
|
||||||
|
"interval_minutes": max(1, int(config.get("interval_minutes", 1) or 1)),
|
||||||
|
"channels": [
|
||||||
|
str(item).strip()
|
||||||
|
for item in config.get("channels", [])
|
||||||
|
if str(item).strip()
|
||||||
|
],
|
||||||
|
"run_all_tracked": bool(config.get("run_all_tracked", True)),
|
||||||
|
}
|
||||||
|
|
||||||
|
return self.update(mutate)["continuous_scraping"]
|
||||||
|
|
||||||
|
def _merge_defaults(self, state: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
merged = deepcopy(self.defaults)
|
||||||
|
for key, value in state.items():
|
||||||
|
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
||||||
|
nested = deepcopy(merged[key])
|
||||||
|
nested.update(value)
|
||||||
|
merged[key] = nested
|
||||||
|
else:
|
||||||
|
merged[key] = value
|
||||||
|
return merged
|
||||||
|
|
||||||
|
|
||||||
|
# ── global state helpers ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
_GLOBAL_STORE: Optional[StateStore] = None
|
||||||
|
|
||||||
|
|
||||||
|
def get_global_store(data_dir: Path) -> StateStore:
|
||||||
|
global _GLOBAL_STORE
|
||||||
|
if _GLOBAL_STORE is None:
|
||||||
|
_GLOBAL_STORE = StateStore(data_dir / "state.json", defaults=GLOBAL_DEFAULTS)
|
||||||
|
return _GLOBAL_STORE
|
||||||
|
|
||||||
|
|
||||||
|
def load_global(data_dir: Path) -> Dict[str, Any]:
|
||||||
|
return get_global_store(data_dir).load()
|
||||||
|
|
||||||
|
|
||||||
|
def save_global(data_dir: Path, state: Dict[str, Any]) -> None:
|
||||||
|
get_global_store(data_dir).save(state)
|
||||||
|
|
||||||
|
|
||||||
|
def list_accounts(data_dir: Path) -> List[str]:
|
||||||
|
return list(load_global(data_dir).get("accounts", []))
|
||||||
|
|
||||||
|
|
||||||
|
def account_exists(data_dir: Path, account_id: str) -> bool:
|
||||||
|
return account_id in list_accounts(data_dir)
|
||||||
|
|
||||||
|
|
||||||
|
# ── per-account state helpers ─────────────────────────────────────────
|
||||||
|
|
||||||
|
_ACCOUNT_STORES: Dict[str, StateStore] = {}
|
||||||
|
_account_stores_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def get_account_store(data_dir: Path, account_id: str) -> StateStore:
|
||||||
|
global _ACCOUNT_STORES
|
||||||
|
with _account_stores_lock:
|
||||||
|
key = f"{data_dir}:{account_id}"
|
||||||
|
if key not in _ACCOUNT_STORES:
|
||||||
|
store = StateStore(
|
||||||
|
data_dir / "accounts" / account_id / "state.json",
|
||||||
|
defaults=ACCOUNT_DEFAULTS,
|
||||||
|
)
|
||||||
|
_ACCOUNT_STORES[key] = store
|
||||||
|
return _ACCOUNT_STORES[key]
|
||||||
|
|
||||||
|
|
||||||
|
def load_account(data_dir: Path, account_id: str) -> Dict[str, Any]:
|
||||||
|
return get_account_store(data_dir, account_id).load()
|
||||||
|
|
||||||
|
|
||||||
|
def save_account(data_dir: Path, account_id: str, state: Dict[str, Any]) -> None:
|
||||||
|
get_account_store(data_dir, account_id).save(state)
|
||||||
|
|
||||||
|
|
||||||
|
def account_data_dir(data_dir: Path, account_id: str) -> Path:
|
||||||
|
"""Per-account directory for channel DBs, media, exports."""
|
||||||
|
return data_dir / "accounts" / account_id
|
||||||
|
|
||||||
|
|
||||||
|
def account_session_path(session_dir: Path, account_id: str) -> str:
|
||||||
|
"""Per-account Telethon session file. Always under session/<id>.session."""
|
||||||
|
return str(session_dir / f"{account_id}.session")
|
||||||
|
|
||||||
|
|
||||||
|
# ── MIGRATION (with data copy) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_channel_data(src_root: Path, dst_root: Path, channel_id: str) -> None:
|
||||||
|
"""Copy a single channel's DB + media from src_root to dst_root."""
|
||||||
|
src_ch = src_root / channel_id
|
||||||
|
dst_ch = dst_root / channel_id
|
||||||
|
if not src_ch.exists():
|
||||||
|
return
|
||||||
|
dst_ch.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# SQLite DB
|
||||||
|
db_name = f"{channel_id}.db"
|
||||||
|
src_db = src_ch / db_name
|
||||||
|
dst_db = dst_ch / db_name
|
||||||
|
if src_db.exists() and not dst_db.exists():
|
||||||
|
shutil.copy2(src_db, dst_db)
|
||||||
|
logger.info(" copied %s", db_name)
|
||||||
|
|
||||||
|
# Media directory
|
||||||
|
src_media = src_ch / "media"
|
||||||
|
dst_media = dst_ch / "media"
|
||||||
|
if src_media.exists() and src_media.is_dir():
|
||||||
|
dst_media.mkdir(parents=True, exist_ok=True)
|
||||||
|
for item in src_media.iterdir():
|
||||||
|
if item.is_file():
|
||||||
|
dst_file = dst_media / item.name
|
||||||
|
if not dst_file.exists():
|
||||||
|
shutil.copy2(item, dst_file)
|
||||||
|
|
||||||
|
# Exports (csv, json)
|
||||||
|
for ext in (".csv", ".json"):
|
||||||
|
for f in src_ch.glob(f"*{ext}"):
|
||||||
|
dst_f = dst_ch / f.name
|
||||||
|
if not dst_f.exists():
|
||||||
|
shutil.copy2(f, dst_f)
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_legacy_state(data_dir: Path, session_dir: Path) -> bool:
|
||||||
|
"""
|
||||||
|
Run once at startup. Detects legacy data/state.json that has api_id
|
||||||
|
and migrates to multi-account format, COPYING channel DBs/media from
|
||||||
|
data/<channel_id>/ → data/accounts/default/<channel_id>/ .
|
||||||
|
|
||||||
|
Returns True if migration was performed.
|
||||||
|
"""
|
||||||
|
state_path = data_dir / "state.json"
|
||||||
|
if not state_path.exists():
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
with state_path.open("r", encoding="utf-8") as f:
|
||||||
|
raw: Dict[str, Any] = json.load(f)
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Already migrated or fresh install with no legacy api_id
|
||||||
|
if raw.get("version") == 2 or not raw.get("api_id"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"=== Legacy state detected — migrating to multi-account format ==="
|
||||||
|
)
|
||||||
|
logger.info("This may take a while if you have many channels with media.")
|
||||||
|
|
||||||
|
# ── 1. Create per-account state for "default" ──────────────────────
|
||||||
|
acc_dir = data_dir / "accounts" / "default"
|
||||||
|
acc_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
acc_state = {
|
||||||
|
"label": "Default",
|
||||||
|
"api_id": raw.get("api_id"),
|
||||||
|
"api_hash": raw.get("api_hash"),
|
||||||
|
"channels": raw.get("channels", {}),
|
||||||
|
"channel_names": raw.get("channel_names", {}),
|
||||||
|
"scrape_media": raw.get("scrape_media", True),
|
||||||
|
"forwarding_rules": raw.get("forwarding_rules", []),
|
||||||
|
"continuous_scraping": raw.get(
|
||||||
|
"continuous_scraping",
|
||||||
|
{
|
||||||
|
"enabled": True,
|
||||||
|
"interval_minutes": 1,
|
||||||
|
"channels": [],
|
||||||
|
"run_all_tracked": True,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
acc_state_path = acc_dir / "state.json"
|
||||||
|
with open(acc_state_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(acc_state, f, ensure_ascii=False, indent=2)
|
||||||
|
f.write("\n")
|
||||||
|
logger.info(" ✓ data/accounts/default/state.json created")
|
||||||
|
|
||||||
|
# ── 2. Copy channel data ───────────────────────────────────────────
|
||||||
|
legacy_channels = list(raw.get("channels", {}).keys())
|
||||||
|
if legacy_channels:
|
||||||
|
logger.info(
|
||||||
|
" Copying %d channel(s) from data/<id>/ → data/accounts/default/<id>/ ...",
|
||||||
|
len(legacy_channels),
|
||||||
|
)
|
||||||
|
for i, channel_id in enumerate(legacy_channels, 1):
|
||||||
|
logger.info(" [%d/%d] channel %s", i, len(legacy_channels), channel_id)
|
||||||
|
_copy_channel_data(data_dir, acc_dir, channel_id)
|
||||||
|
logger.info(" ✓ Channel data copied")
|
||||||
|
else:
|
||||||
|
logger.info(" (no tracked channels to copy)")
|
||||||
|
|
||||||
|
# ── 3. Rewrite global state.json ───────────────────────────────────
|
||||||
|
global_state = {
|
||||||
|
"accounts": ["default"],
|
||||||
|
"version": 2,
|
||||||
|
}
|
||||||
|
with open(state_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(global_state, f, ensure_ascii=False, indent=2)
|
||||||
|
f.write("\n")
|
||||||
|
logger.info(" ✓ data/state.json rewritten (accounts list only)")
|
||||||
|
|
||||||
|
# ── 4. Rename legacy session file ─────────────────────────────────
|
||||||
|
legacy_session = session_dir / "session.session"
|
||||||
|
if legacy_session.exists():
|
||||||
|
new_session = session_dir / "default.session"
|
||||||
|
if not new_session.exists():
|
||||||
|
legacy_session.rename(new_session)
|
||||||
|
logger.info(" ✓ session/session.session → session/default.session")
|
||||||
|
|
||||||
|
logger.info("=== Migration complete ===")
|
||||||
|
return True
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
services:
|
||||||
|
telegram-scraper:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
container_name: telegram-scraper
|
||||||
|
image: gcr.forust.xyz/forust/telegram-scraper:latest
|
||||||
|
tty: true
|
||||||
|
user: '1000:1000'
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- 7887:8080
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
CMD,
|
||||||
|
python,
|
||||||
|
-c,
|
||||||
|
"import http.client, json; c=http.client.HTTPConnection('localhost', 8080); c.request('GET', '/health'); r=c.getresponse(); exit(0) if json.loads(r.read()).get('ok') else exit(1)",
|
||||||
|
]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 15s
|
||||||
|
volumes:
|
||||||
|
- ./data:/app/data
|
||||||
|
- ./session:/app/session
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
import sqlite3
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from app_state import StateStore
|
||||||
|
|
||||||
|
|
||||||
|
def health_payload(
|
||||||
|
data_dir: Path,
|
||||||
|
session_dir: Path,
|
||||||
|
state_store: StateStore,
|
||||||
|
continuous_snapshot: Dict[str, Any],
|
||||||
|
job_queue_size: int,
|
||||||
|
account_ids: Optional[List[str]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Return a health-check payload for the application.
|
||||||
|
|
||||||
|
NOTE: This endpoint does NOT support message search or query parameters.
|
||||||
|
Health concerns itself with filesystem, database connectivity, job queue,
|
||||||
|
and account session state. Message search is handled by the message API
|
||||||
|
endpoints (/api/channels/*/messages, /api/accounts/*/channels/*/messages).
|
||||||
|
"""
|
||||||
|
checks = {
|
||||||
|
"data_dir": _dir_check(data_dir, writable=True),
|
||||||
|
"session_dir": _dir_check(session_dir, writable=True),
|
||||||
|
"state_file": _state_check(state_store),
|
||||||
|
"sqlite": _sqlite_check(),
|
||||||
|
"continuous": _continuous_check(continuous_snapshot),
|
||||||
|
"job_queue": {"ok": True, "size": job_queue_size},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Per-account health
|
||||||
|
if account_ids:
|
||||||
|
account_checks: Dict[str, Any] = {}
|
||||||
|
for acc_id in account_ids:
|
||||||
|
acc_dir = data_dir / "accounts" / acc_id
|
||||||
|
session_file = session_dir / f"{acc_id}.session"
|
||||||
|
account_checks[acc_id] = {
|
||||||
|
"data_dir": _dir_check(acc_dir),
|
||||||
|
"session_file": {
|
||||||
|
"ok": session_file.exists(),
|
||||||
|
"path": str(session_file),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
checks["accounts"] = {
|
||||||
|
"ok": all(
|
||||||
|
item.get("data_dir", {}).get("ok", False)
|
||||||
|
for item in account_checks.values()
|
||||||
|
),
|
||||||
|
"items": account_checks,
|
||||||
|
}
|
||||||
|
|
||||||
|
ok = all(item.get("ok", False) for item in checks.values())
|
||||||
|
return {"ok": ok, "status": "ok" if ok else "degraded", "checks": checks}
|
||||||
|
|
||||||
|
|
||||||
|
def _dir_check(path: Path, writable: bool = False) -> Dict[str, Any]:
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
ok = path.exists() and path.is_dir()
|
||||||
|
payload: Dict[str, Any] = {"ok": ok, "path": str(path)}
|
||||||
|
if writable:
|
||||||
|
probe = path / ".healthcheck"
|
||||||
|
try:
|
||||||
|
probe.write_text("ok", encoding="utf-8")
|
||||||
|
probe.unlink(missing_ok=True)
|
||||||
|
payload["writable"] = True
|
||||||
|
except OSError as exc:
|
||||||
|
payload.update({"ok": False, "writable": False, "error": str(exc)})
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _state_check(state_store: StateStore) -> Dict[str, Any]:
|
||||||
|
try:
|
||||||
|
state = state_store.load()
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"path": str(state_store.path),
|
||||||
|
"has_api_credentials": bool(state.get("api_id") and state.get("api_hash")),
|
||||||
|
"tracked_channels": len(state.get("channels", {})),
|
||||||
|
}
|
||||||
|
except Exception as exc:
|
||||||
|
return {"ok": False, "path": str(state_store.path), "error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _sqlite_check() -> Dict[str, Any]:
|
||||||
|
try:
|
||||||
|
conn = sqlite3.connect(":memory:")
|
||||||
|
conn.execute("SELECT 1")
|
||||||
|
conn.close()
|
||||||
|
return {"ok": True}
|
||||||
|
except sqlite3.Error as exc:
|
||||||
|
return {"ok": False, "error": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _continuous_check(snapshot: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
running_accounts = snapshot.get("running_accounts")
|
||||||
|
if running_accounts is None:
|
||||||
|
running_accounts = snapshot
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"account_count": len(running_accounts),
|
||||||
|
"accounts": {
|
||||||
|
aid: {
|
||||||
|
"running": bool((info.get("status") or info).get("running")),
|
||||||
|
"enabled": bool((info.get("config") or {}).get("enabled", False)),
|
||||||
|
"last_error": (info.get("status") or info).get("last_error"),
|
||||||
|
}
|
||||||
|
for aid, info in running_accounts.items()
|
||||||
|
if isinstance(info, dict)
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: telegram-scraper
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: telegram-scraper-service
|
||||||
|
namespace: telegram-scraper
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: telegram-scraper
|
||||||
|
ports:
|
||||||
|
- port: 8080
|
||||||
|
targetPort: 8080
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: telegram-scraper-statefulset
|
||||||
|
namespace: telegram-scraper
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: telegram-scraper
|
||||||
|
serviceName: telegram-scraper-service
|
||||||
|
replicas: 1
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: telegram-scraper
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: telegram-scraper
|
||||||
|
image: gcr.forust.xyz/forust/telegram-scraper:latest
|
||||||
|
stdin: true
|
||||||
|
tty: true
|
||||||
|
ports:
|
||||||
|
- containerPort: 8080
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 8080
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 30
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 8080
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 10
|
||||||
|
volumeMounts:
|
||||||
|
- name: telegram-scraper-data
|
||||||
|
mountPath: /app/data
|
||||||
|
- name: telegram-scraper-session
|
||||||
|
mountPath: /app/session
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: telegram-scraper-data
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
|
- metadata:
|
||||||
|
name: telegram-scraper-session
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 5Mi
|
||||||
|
---
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: telegram-scraper-route
|
||||||
|
namespace: telegram-scraper
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- websecure
|
||||||
|
routes:
|
||||||
|
- match: Host(`tg.workstation.internal`)
|
||||||
|
kind: Rule
|
||||||
|
services:
|
||||||
|
- name: telegram-scraper-service
|
||||||
|
port: 8080
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from webui_server import run_server
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||||
|
datefmt="%Y-%m-%d %H:%M:%S",
|
||||||
|
stream=sys.stdout,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Run legacy migration before starting the server
|
||||||
|
from app_state import migrate_legacy_state
|
||||||
|
|
||||||
|
data_dir = Path("data")
|
||||||
|
session_dir = Path("session")
|
||||||
|
try:
|
||||||
|
if migrate_legacy_state(data_dir, session_dir):
|
||||||
|
logger.info("Legacy migration completed successfully.")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Migration failed: %s", exc)
|
||||||
|
|
||||||
|
run_server()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
[project]
|
||||||
|
name = "telegram-scraper"
|
||||||
|
version = "2.0.0"
|
||||||
|
description = "Telegram-scraper with webui"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.11"
|
||||||
|
dependencies = [
|
||||||
|
"aiohappyeyeballs==2.6.1",
|
||||||
|
"aiohttp==3.12.14",
|
||||||
|
"aiosignal==1.4.0",
|
||||||
|
"attrs==25.3.0",
|
||||||
|
"frozenlist==1.7.0",
|
||||||
|
"idna==3.10",
|
||||||
|
"multidict==6.6.3",
|
||||||
|
"propcache==0.3.2",
|
||||||
|
"pyaes==1.6.1",
|
||||||
|
"pyasn1==0.6.1",
|
||||||
|
"qrcode==8.0",
|
||||||
|
"rsa==4.9.1",
|
||||||
|
"Telethon==1.40.0",
|
||||||
|
"yarl==1.20.1",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
pythonpath = ["."]
|
||||||
+1
-2
@@ -1,8 +1,7 @@
|
|||||||
aiohappyeyeballs==2.6.1
|
aiohappyeyeballs==2.6.1
|
||||||
aiohttp==3.12.14
|
aiohttp==3.12.14
|
||||||
aiosignal==1.4.0
|
aiosignal==1.4.0
|
||||||
asyncio==3.4.3
|
attrs==25.3.0
|
||||||
attrs==25.3.0
|
|
||||||
frozenlist==1.7.0
|
frozenlist==1.7.0
|
||||||
idna==3.10
|
idna==3.10
|
||||||
multidict==6.6.3
|
multidict==6.6.3
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from app_state import StateStore
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ScraperJobService:
|
||||||
|
def __init__(self, state_store: StateStore):
|
||||||
|
self.state_store = state_store
|
||||||
|
|
||||||
|
def run(self, job_type: str, payload: Dict[str, Any]) -> None:
|
||||||
|
if job_type == "set_scrape_media":
|
||||||
|
value = bool(payload["value"])
|
||||||
|
|
||||||
|
def mutate(state: Dict[str, Any]) -> None:
|
||||||
|
state["scrape_media"] = value
|
||||||
|
|
||||||
|
self.state_store.update(mutate)
|
||||||
|
logger.info("Media scraping set to %s", value)
|
||||||
|
return
|
||||||
|
|
||||||
|
asyncio.run(self._run_async(job_type, payload))
|
||||||
|
|
||||||
|
async def _run_async(self, job_type: str, payload: Dict[str, Any]) -> None:
|
||||||
|
# Extract account_id from payload, default to None (legacy)
|
||||||
|
account_id: Optional[str] = payload.get("account_id")
|
||||||
|
ScraperClass = self._import_scraper_class()
|
||||||
|
scraper = ScraperClass(account_id=account_id)
|
||||||
|
|
||||||
|
if account_id:
|
||||||
|
from app_state import load_account
|
||||||
|
acc_state = load_account(self.state_store.path.parent, account_id)
|
||||||
|
# Load per-account state into scraper state
|
||||||
|
scraper.state = acc_state
|
||||||
|
|
||||||
|
initialized = await scraper.initialize_client(interactive=False)
|
||||||
|
if not initialized:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Telegram client is not ready. Check credentials, session, and write access."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
if job_type == "scrape_channel":
|
||||||
|
await self._scrape_channels(scraper, [payload["channel_id"]])
|
||||||
|
elif job_type == "scrape_all":
|
||||||
|
await self._scrape_channels(
|
||||||
|
scraper, list(scraper.state.get("channels", {}).keys())
|
||||||
|
)
|
||||||
|
elif job_type == "scrape_selected":
|
||||||
|
await self._scrape_channels(
|
||||||
|
scraper,
|
||||||
|
[str(channel_id) for channel_id in payload.get("channels", [])],
|
||||||
|
)
|
||||||
|
elif job_type == "export_all":
|
||||||
|
await scraper.export_data()
|
||||||
|
elif job_type == "export_channel":
|
||||||
|
channel_id = payload["channel_id"]
|
||||||
|
scraper.export_to_csv(channel_id)
|
||||||
|
scraper.export_to_json(channel_id)
|
||||||
|
elif job_type == "rescrape_media":
|
||||||
|
await scraper.rescrape_media(payload["channel_id"])
|
||||||
|
elif job_type == "fix_missing_media":
|
||||||
|
await scraper.fix_missing_media(payload["channel_id"])
|
||||||
|
elif job_type == "refresh_dialogs":
|
||||||
|
await scraper.list_channels()
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Unsupported job type: {job_type}")
|
||||||
|
finally:
|
||||||
|
scraper.close_db_connections()
|
||||||
|
if scraper.client:
|
||||||
|
await scraper.client.disconnect()
|
||||||
|
|
||||||
|
async def _scrape_channels(self, scraper, channels: List[str]) -> None:
|
||||||
|
for channel_id in channels:
|
||||||
|
offset = int(scraper.state.get("channels", {}).get(channel_id, 0) or 0)
|
||||||
|
await scraper.scrape_channel(channel_id, offset)
|
||||||
|
|
||||||
|
def _import_scraper_class(self):
|
||||||
|
from telegram_scraper_with_forwarding import OptimizedTelegramScraper
|
||||||
|
|
||||||
|
return OptimizedTelegramScraper
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
PORT = os.environ.get("TELEGRAM_SCRAPER_SMOKE_PORT", "18080")
|
||||||
|
BASE_URL = f"http://127.0.0.1:{PORT}"
|
||||||
|
ENDPOINTS = [
|
||||||
|
"/",
|
||||||
|
"/viewer",
|
||||||
|
"/swagger",
|
||||||
|
"/openapi.json",
|
||||||
|
"/health",
|
||||||
|
"/health/continuous",
|
||||||
|
"/api/dashboard",
|
||||||
|
"/api/continuous",
|
||||||
|
"/api/channels",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def fetch(path: str) -> tuple[int, bytes]:
|
||||||
|
with urllib.request.urlopen(BASE_URL + path, timeout=3) as response:
|
||||||
|
return response.status, response.read()
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_server(process: subprocess.Popen) -> None:
|
||||||
|
deadline = time.time() + 15
|
||||||
|
last_error = None
|
||||||
|
while time.time() < deadline:
|
||||||
|
if process.poll() is not None:
|
||||||
|
raise RuntimeError(f"server exited with code {process.returncode}")
|
||||||
|
try:
|
||||||
|
fetch("/health")
|
||||||
|
return
|
||||||
|
except Exception as exc:
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(0.3)
|
||||||
|
raise RuntimeError(f"server did not become ready: {last_error}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
env = {
|
||||||
|
**os.environ,
|
||||||
|
"TELEGRAM_SCRAPER_PORT": PORT,
|
||||||
|
"TELEGRAM_SCRAPER_START_CONTINUOUS": "0",
|
||||||
|
}
|
||||||
|
process = subprocess.Popen(
|
||||||
|
[sys.executable, "main.py"],
|
||||||
|
env=env,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
wait_for_server(process)
|
||||||
|
for endpoint in ENDPOINTS:
|
||||||
|
status, body = fetch(endpoint)
|
||||||
|
if status != 200:
|
||||||
|
raise RuntimeError(f"{endpoint} returned HTTP {status}")
|
||||||
|
if (
|
||||||
|
endpoint.endswith(".json")
|
||||||
|
or endpoint.startswith("/api")
|
||||||
|
or endpoint.startswith("/health")
|
||||||
|
):
|
||||||
|
json.loads(body.decode("utf-8"))
|
||||||
|
print(f"ok {endpoint}")
|
||||||
|
return 0
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"smoke test failed: {exc}", file=sys.stderr)
|
||||||
|
if process.stdout:
|
||||||
|
print(process.stdout.read(), file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
finally:
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
process.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
process.kill()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
+426
-250
File diff suppressed because it is too large
Load Diff
+811
-507
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,638 @@
|
|||||||
|
"""
|
||||||
|
Smoke / integration tests for the multi-account Telegram scraper core logic.
|
||||||
|
|
||||||
|
These tests do NOT require Telegram credentials or network access.
|
||||||
|
They verify:
|
||||||
|
- Account creation and state isolation
|
||||||
|
- Account import safety (no auto-start continuous)
|
||||||
|
- Account export round-trip
|
||||||
|
- Account deletion (sidecar file cleanup)
|
||||||
|
- Active account persistence helpers
|
||||||
|
- Message search SQL construction
|
||||||
|
- Health payload structure
|
||||||
|
- Continuous orchestrator worker deduplication
|
||||||
|
- load_messages() pagination with search filter
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
# ── Mock heavy dependencies before any webui_server import ─────────────
|
||||||
|
# We need to mock qrcode and telethon because they aren't installed in CI.
|
||||||
|
sys.modules["qrcode"] = MagicMock()
|
||||||
|
sys.modules["qrcode.image"] = MagicMock()
|
||||||
|
sys.modules["qrcode.image.svg"] = MagicMock()
|
||||||
|
sys.modules["telethon"] = MagicMock()
|
||||||
|
sys.modules["telethon.errors"] = MagicMock()
|
||||||
|
sys.modules["telethon.errors"].SessionPasswordNeededError = type("SessionPasswordNeededError", (Exception,), {})
|
||||||
|
sys.modules["telegram_scraper_with_forwarding"] = MagicMock()
|
||||||
|
sys.modules["telegram_scraper_with_forwarding"]._ensure_session_wal = lambda p: None
|
||||||
|
sys.modules["telegram_scraper_with_forwarding"].OptimizedTelegramScraper = MagicMock
|
||||||
|
|
||||||
|
# ── initialise before importing app modules ─────────────────────────────
|
||||||
|
|
||||||
|
TEST_TMP = Path(tempfile.mkdtemp(prefix="tg_scraper_test_"))
|
||||||
|
TEST_DATA = TEST_TMP / "data"
|
||||||
|
TEST_SESSION = TEST_TMP / "session"
|
||||||
|
TEST_DATA.mkdir(parents=True, exist_ok=True)
|
||||||
|
TEST_SESSION.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
os.environ["TELEGRAM_SCRAPER_HOST"] = "127.0.0.1"
|
||||||
|
os.environ["TELEGRAM_SCRAPER_PORT"] = "0"
|
||||||
|
os.environ["TELEGRAM_SCRAPER_START_CONTINUOUS"] = "0"
|
||||||
|
|
||||||
|
# Now safe to import app modules
|
||||||
|
import app_state # noqa: E402
|
||||||
|
from app_state import ( # noqa: E402
|
||||||
|
ACCOUNT_DEFAULTS,
|
||||||
|
account_data_dir,
|
||||||
|
account_exists,
|
||||||
|
account_session_path,
|
||||||
|
get_account_store,
|
||||||
|
get_global_store,
|
||||||
|
list_accounts,
|
||||||
|
load_account,
|
||||||
|
)
|
||||||
|
from health import health_payload # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _init_test_env():
|
||||||
|
"""Reset global caches and re-initialise for a clean test."""
|
||||||
|
app_state._GLOBAL_STORE = None
|
||||||
|
with app_state._account_stores_lock:
|
||||||
|
app_state._ACCOUNT_STORES.clear()
|
||||||
|
|
||||||
|
|
||||||
|
_init_test_env()
|
||||||
|
|
||||||
|
|
||||||
|
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def make_account_id() -> str:
|
||||||
|
return f"test-{int(time.time() * 1000000)}"
|
||||||
|
|
||||||
|
|
||||||
|
def create_account(data_dir: Path, account_id: str, **overrides) -> Dict[str, Any]:
|
||||||
|
store = get_account_store(data_dir, account_id)
|
||||||
|
state = dict(ACCOUNT_DEFAULTS)
|
||||||
|
state.update(overrides)
|
||||||
|
state["label"] = overrides.get("label", account_id)
|
||||||
|
store.save(state)
|
||||||
|
|
||||||
|
def mutate(g: Dict[str, Any]) -> None:
|
||||||
|
accounts = g.setdefault("accounts", [])
|
||||||
|
if account_id not in accounts:
|
||||||
|
accounts.append(account_id)
|
||||||
|
|
||||||
|
get_global_store(data_dir).update(mutate)
|
||||||
|
return load_account(data_dir, account_id)
|
||||||
|
|
||||||
|
|
||||||
|
def create_channel_db(data_dir: Path, account_id: Optional[str], channel_id: str, messages: List[Dict[str, Any]]) -> None:
|
||||||
|
if account_id:
|
||||||
|
db_dir = account_data_dir(data_dir, account_id) / channel_id
|
||||||
|
else:
|
||||||
|
db_dir = data_dir / channel_id
|
||||||
|
db_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
db_path = db_dir / f"{channel_id}.db"
|
||||||
|
conn = sqlite3.connect(str(db_path))
|
||||||
|
conn.execute(
|
||||||
|
"""CREATE TABLE IF NOT EXISTS messages (
|
||||||
|
message_id INTEGER PRIMARY KEY,
|
||||||
|
date TEXT, sender_id INTEGER,
|
||||||
|
first_name TEXT, last_name TEXT, username TEXT,
|
||||||
|
message TEXT, media_type TEXT, media_path TEXT,
|
||||||
|
reply_to INTEGER, post_author TEXT,
|
||||||
|
views INTEGER, forwards INTEGER, reactions TEXT
|
||||||
|
)"""
|
||||||
|
)
|
||||||
|
for m in messages:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO messages (message_id, date, message) VALUES (?, ?, ?)",
|
||||||
|
(m["message_id"], m.get("date"), m.get("message")),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ── Fixture setup / teardown ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def setup_function():
|
||||||
|
_init_test_env()
|
||||||
|
for child in TEST_DATA.iterdir():
|
||||||
|
if child.is_dir():
|
||||||
|
shutil.rmtree(child, ignore_errors=True)
|
||||||
|
else:
|
||||||
|
child.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def teardown_function():
|
||||||
|
_init_test_env()
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
# Tests
|
||||||
|
# ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
|
||||||
|
class TestAccountLifecycle:
|
||||||
|
"""Account creation, isolation, import, export, deletion."""
|
||||||
|
|
||||||
|
def test_create_account_minimal(self):
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(TEST_DATA, aid)
|
||||||
|
assert account_exists(TEST_DATA, aid)
|
||||||
|
assert aid in list_accounts(TEST_DATA)
|
||||||
|
state = load_account(TEST_DATA, aid)
|
||||||
|
assert state["label"] == aid
|
||||||
|
assert state["continuous_scraping"]["enabled"] is False
|
||||||
|
|
||||||
|
def test_account_state_isolation(self):
|
||||||
|
a1, a2 = make_account_id(), make_account_id()
|
||||||
|
create_account(TEST_DATA, a1, label="Alpha", api_id=111)
|
||||||
|
create_account(TEST_DATA, a2, label="Beta", api_id=222)
|
||||||
|
s1 = load_account(TEST_DATA, a1)
|
||||||
|
s2 = load_account(TEST_DATA, a2)
|
||||||
|
assert s1["label"] == "Alpha"
|
||||||
|
assert s1["api_id"] == 111
|
||||||
|
assert s2["label"] == "Beta"
|
||||||
|
assert s2["api_id"] == 222
|
||||||
|
# channels are segregated — mutate one, verify other unchanged
|
||||||
|
s1.setdefault("channels", {})["-100aaa"] = 1
|
||||||
|
get_account_store(TEST_DATA, a1).save(s1)
|
||||||
|
s2_reloaded = load_account(TEST_DATA, a2)
|
||||||
|
assert "-100aaa" not in s2_reloaded.get("channels", {})
|
||||||
|
|
||||||
|
def test_delete_account_cleans_session_sidecars(self):
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(TEST_DATA, aid)
|
||||||
|
session_path = Path(account_session_path(TEST_SESSION, aid))
|
||||||
|
session_path.write_text("session-data")
|
||||||
|
Path(str(session_path) + "-wal").write_text("wal")
|
||||||
|
Path(str(session_path) + "-shm").write_text("shm")
|
||||||
|
Path(str(session_path) + "-journal").write_text("journal")
|
||||||
|
|
||||||
|
acc_dir = account_data_dir(TEST_DATA, aid)
|
||||||
|
if acc_dir.exists():
|
||||||
|
shutil.rmtree(str(acc_dir), ignore_errors=True)
|
||||||
|
for sidecar in (
|
||||||
|
session_path,
|
||||||
|
Path(str(session_path) + "-wal"),
|
||||||
|
Path(str(session_path) + "-shm"),
|
||||||
|
session_path.with_suffix(session_path.suffix + "-journal"),
|
||||||
|
):
|
||||||
|
if sidecar.exists():
|
||||||
|
sidecar.unlink()
|
||||||
|
|
||||||
|
assert not session_path.exists()
|
||||||
|
assert not Path(str(session_path) + "-wal").exists()
|
||||||
|
assert not Path(str(session_path) + "-shm").exists()
|
||||||
|
assert not Path(str(session_path) + "-journal").exists()
|
||||||
|
assert not acc_dir.exists()
|
||||||
|
|
||||||
|
def test_delete_account_skips_missing_sidecars(self):
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(TEST_DATA, aid)
|
||||||
|
session_path = Path(account_session_path(TEST_SESSION, aid))
|
||||||
|
session_path.write_text("session-only")
|
||||||
|
acc_dir = account_data_dir(TEST_DATA, aid)
|
||||||
|
if acc_dir.exists():
|
||||||
|
shutil.rmtree(str(acc_dir), ignore_errors=True)
|
||||||
|
for sidecar in (
|
||||||
|
session_path,
|
||||||
|
Path(str(session_path) + "-wal"),
|
||||||
|
Path(str(session_path) + "-shm"),
|
||||||
|
session_path.with_suffix(session_path.suffix + "-journal"),
|
||||||
|
):
|
||||||
|
if sidecar.exists():
|
||||||
|
sidecar.unlink()
|
||||||
|
assert not session_path.exists()
|
||||||
|
|
||||||
|
def test_import_export_round_trip(self):
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(
|
||||||
|
TEST_DATA, aid,
|
||||||
|
label="Imported",
|
||||||
|
api_id=999,
|
||||||
|
api_hash="abc123",
|
||||||
|
channels={"-100ch1": 42},
|
||||||
|
scrape_media=False,
|
||||||
|
continuous_scraping={"enabled": True, "interval_minutes": 5, "channels": [], "run_all_tracked": True},
|
||||||
|
)
|
||||||
|
state = load_account(TEST_DATA, aid)
|
||||||
|
|
||||||
|
aid2 = make_account_id()
|
||||||
|
raw_state = state
|
||||||
|
imported_state = {
|
||||||
|
"label": raw_state.get("label", aid2),
|
||||||
|
"api_id": raw_state.get("api_id"),
|
||||||
|
"api_hash": raw_state.get("api_hash"),
|
||||||
|
"channels": raw_state.get("channels", {}),
|
||||||
|
"channel_names": raw_state.get("channel_names", {}),
|
||||||
|
"scrape_media": bool(raw_state.get("scrape_media", True)),
|
||||||
|
"forwarding_rules": raw_state.get("forwarding_rules", []),
|
||||||
|
"continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else {
|
||||||
|
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
get_account_store(TEST_DATA, aid2).save(imported_state)
|
||||||
|
|
||||||
|
def mutate_global(g):
|
||||||
|
g.setdefault("accounts", []).append(aid2)
|
||||||
|
get_global_store(TEST_DATA).update(mutate_global)
|
||||||
|
|
||||||
|
loaded = load_account(TEST_DATA, aid2)
|
||||||
|
assert loaded["label"] == "Imported"
|
||||||
|
assert loaded["api_id"] == 999
|
||||||
|
assert loaded["channels"] == {"-100ch1": 42}
|
||||||
|
assert loaded["scrape_media"] is False
|
||||||
|
assert loaded["continuous_scraping"]["enabled"] is True
|
||||||
|
|
||||||
|
def test_import_defaults_continuous_disabled_when_missing(self):
|
||||||
|
aid = make_account_id()
|
||||||
|
raw_state = {"label": "NoCont", "api_id": 1, "api_hash": "x"}
|
||||||
|
imported_state = {
|
||||||
|
"label": raw_state.get("label", aid),
|
||||||
|
"api_id": raw_state.get("api_id"),
|
||||||
|
"api_hash": raw_state.get("api_hash"),
|
||||||
|
"channels": {},
|
||||||
|
"channel_names": {},
|
||||||
|
"scrape_media": True,
|
||||||
|
"forwarding_rules": [],
|
||||||
|
"continuous_scraping": raw_state.get("continuous_scraping") if isinstance(raw_state.get("continuous_scraping"), dict) else {
|
||||||
|
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
get_account_store(TEST_DATA, aid).save(imported_state)
|
||||||
|
|
||||||
|
def mutate_global(g):
|
||||||
|
g.setdefault("accounts", []).append(aid)
|
||||||
|
get_global_store(TEST_DATA).update(mutate_global)
|
||||||
|
|
||||||
|
loaded = load_account(TEST_DATA, aid)
|
||||||
|
assert loaded["continuous_scraping"]["enabled"] is False
|
||||||
|
assert loaded["continuous_scraping"]["run_all_tracked"] is True
|
||||||
|
|
||||||
|
|
||||||
|
class TestInputSafety:
|
||||||
|
"""Small safety checks for values that flow into files or exports."""
|
||||||
|
|
||||||
|
def test_channel_id_rejects_path_characters(self):
|
||||||
|
import webui_server as ws_module
|
||||||
|
|
||||||
|
assert ws_module.normalize_channel_id("@valid_name") == "valid_name"
|
||||||
|
for bad in ("../escape", "nested/channel", r"nested\channel", ".", "..", ""):
|
||||||
|
try:
|
||||||
|
ws_module.normalize_channel_id(bad)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
raise AssertionError(f"accepted unsafe channel_id: {bad!r}")
|
||||||
|
|
||||||
|
def test_export_account_state_redacts_api_hash_by_default(self):
|
||||||
|
import webui_server as ws_module
|
||||||
|
|
||||||
|
state = {"label": "Work", "api_id": 123, "api_hash": "secret", "channels": {}}
|
||||||
|
redacted = ws_module.export_account_state(state)
|
||||||
|
assert redacted["api_hash"] is None
|
||||||
|
assert redacted["api_hash_present"] is True
|
||||||
|
assert state["api_hash"] == "secret"
|
||||||
|
|
||||||
|
full = ws_module.export_account_state(state, include_secrets=True)
|
||||||
|
assert full["api_hash"] == "secret"
|
||||||
|
assert "api_hash_present" not in full
|
||||||
|
|
||||||
|
|
||||||
|
class TestContinuousOrchestrator:
|
||||||
|
"""Continuous scraping manager safety using mocked webui_server."""
|
||||||
|
|
||||||
|
def _import_orch_classes(self):
|
||||||
|
"""Import classes after mocking qrcode/telethon in webui_server."""
|
||||||
|
import webui_server as ws_module
|
||||||
|
return ws_module.PerAccountContinuousScrapeManager, ws_module.ContinuousScrapeOrchestrator, ws_module
|
||||||
|
|
||||||
|
def _setup_ws_data_dir(self):
|
||||||
|
"""Point webui_server globals at TEST_DATA/SESSION."""
|
||||||
|
import webui_server as ws_module
|
||||||
|
self._ws_orig_data = ws_module.DATA_DIR
|
||||||
|
self._ws_orig_session = ws_module.SESSION_DIR
|
||||||
|
ws_module.DATA_DIR = TEST_DATA
|
||||||
|
ws_module.SESSION_DIR = TEST_SESSION
|
||||||
|
ws_module.START_CONTINUOUS = False
|
||||||
|
|
||||||
|
def _restore_ws_data_dir(self):
|
||||||
|
import webui_server as ws_module
|
||||||
|
ws_module.DATA_DIR = self._ws_orig_data
|
||||||
|
ws_module.SESSION_DIR = self._ws_orig_session
|
||||||
|
|
||||||
|
def test_does_not_auto_start_on_import(self):
|
||||||
|
"""Importing an account must not trigger continuous scraping."""
|
||||||
|
PerAccountContinuousScrapeManager, ContinuousScrapeOrchestrator, ws = self._import_orch_classes()
|
||||||
|
self._setup_ws_data_dir()
|
||||||
|
try:
|
||||||
|
orch = ContinuousScrapeOrchestrator()
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(TEST_DATA, aid, continuous_scraping={
|
||||||
|
"enabled": True, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
|
||||||
|
})
|
||||||
|
orch.add_account(aid, auto_start=False)
|
||||||
|
snap = orch.snapshot_for(aid)
|
||||||
|
assert snap["status"]["running"] is False
|
||||||
|
finally:
|
||||||
|
self._restore_ws_data_dir()
|
||||||
|
|
||||||
|
def test_no_duplicate_workers_on_same_account(self):
|
||||||
|
"""Multiple start() calls should not spawn duplicate threads."""
|
||||||
|
PerAccountContinuousScrapeManager, _, ws = self._import_orch_classes()
|
||||||
|
self._setup_ws_data_dir()
|
||||||
|
try:
|
||||||
|
aid = make_account_id()
|
||||||
|
# Must set enabled=True or refresh_config() will stop the thread
|
||||||
|
create_account(TEST_DATA, aid, continuous_scraping={
|
||||||
|
"enabled": True, "interval_minutes": 60, "channels": [], "run_all_tracked": True,
|
||||||
|
})
|
||||||
|
mgr = PerAccountContinuousScrapeManager(aid)
|
||||||
|
# Override refresh_config to prevent auth check from interfering
|
||||||
|
original_refresh = mgr.refresh_config
|
||||||
|
mgr.refresh_config = lambda: None
|
||||||
|
mgr.start()
|
||||||
|
t1 = mgr.thread
|
||||||
|
mgr.start() # second call — should be no-op
|
||||||
|
t2 = mgr.thread
|
||||||
|
assert t1 is t2, "start() spawned a second thread"
|
||||||
|
# Also verify only one thread is alive for this manager
|
||||||
|
alive_count = sum(
|
||||||
|
1 for t in threading.enumerate()
|
||||||
|
if t is t1 or t is t2
|
||||||
|
)
|
||||||
|
assert alive_count <= 1, "duplicate threads detected"
|
||||||
|
mgr.stop()
|
||||||
|
mgr.refresh_config = original_refresh
|
||||||
|
finally:
|
||||||
|
self._restore_ws_data_dir()
|
||||||
|
|
||||||
|
def test_disabled_account_not_started(self):
|
||||||
|
"""start_all() must not start accounts with enabled=False."""
|
||||||
|
_, ContinuousScrapeOrchestrator, ws = self._import_orch_classes()
|
||||||
|
self._setup_ws_data_dir()
|
||||||
|
try:
|
||||||
|
orch = ContinuousScrapeOrchestrator()
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(TEST_DATA, aid, continuous_scraping={
|
||||||
|
"enabled": False, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
|
||||||
|
})
|
||||||
|
orch.start_all()
|
||||||
|
snap = orch.snapshot_for(aid)
|
||||||
|
assert snap["status"]["running"] is False
|
||||||
|
finally:
|
||||||
|
self._restore_ws_data_dir()
|
||||||
|
|
||||||
|
|
||||||
|
class TestMessageSearch:
|
||||||
|
"""load_messages() SQL correctness with pagination and search."""
|
||||||
|
|
||||||
|
def _import_load_messages(self):
|
||||||
|
import webui_server as ws_module
|
||||||
|
# Point at TEST_DATA
|
||||||
|
self._ws_orig = ws_module.DATA_DIR
|
||||||
|
ws_module.DATA_DIR = TEST_DATA
|
||||||
|
return ws_module.load_messages
|
||||||
|
|
||||||
|
def _restore(self):
|
||||||
|
import webui_server as ws_module
|
||||||
|
ws_module.DATA_DIR = self._ws_orig
|
||||||
|
|
||||||
|
def test_search_filters_correctly(self):
|
||||||
|
channel_id = "-100searchtest"
|
||||||
|
messages = [
|
||||||
|
{"message_id": 1, "date": "2024-01-01", "message": "hello world"},
|
||||||
|
{"message_id": 2, "date": "2024-01-02", "message": "foo bar baz"},
|
||||||
|
{"message_id": 3, "date": "2024-01-03", "message": "hello again"},
|
||||||
|
]
|
||||||
|
create_channel_db(TEST_DATA, "default", channel_id, messages)
|
||||||
|
load_messages = self._import_load_messages()
|
||||||
|
try:
|
||||||
|
result = load_messages("default", channel_id, limit=100, search="hello")
|
||||||
|
finally:
|
||||||
|
self._restore()
|
||||||
|
texts = [m["text"] for m in result]
|
||||||
|
assert "hello world" in texts
|
||||||
|
assert "hello again" in texts
|
||||||
|
assert "foo bar baz" not in texts
|
||||||
|
|
||||||
|
def test_before_id_pagination_works(self):
|
||||||
|
channel_id = "-100pagination"
|
||||||
|
messages = [
|
||||||
|
{"message_id": i, "date": f"2024-01-{i:02d}", "message": f"msg-{i}"}
|
||||||
|
for i in range(1, 21)
|
||||||
|
]
|
||||||
|
create_channel_db(TEST_DATA, "default", channel_id, messages)
|
||||||
|
load_messages = self._import_load_messages()
|
||||||
|
try:
|
||||||
|
result = load_messages("default", channel_id, limit=5, before_message_id=15)
|
||||||
|
finally:
|
||||||
|
self._restore()
|
||||||
|
assert len(result) == 5
|
||||||
|
ids = [m["message_id"] for m in result]
|
||||||
|
assert all(i < 15 for i in ids)
|
||||||
|
# load_messages returns rows in ascending message_id order (reversed from DESC query)
|
||||||
|
assert ids == [10, 11, 12, 13, 14]
|
||||||
|
|
||||||
|
def test_search_with_pagination(self):
|
||||||
|
channel_id = "-100searchpages"
|
||||||
|
messages = [
|
||||||
|
{"message_id": i, "date": f"2024-01-{i:02d}", "message": f"hello-{i}" if i % 2 else f"other-{i}"}
|
||||||
|
for i in range(1, 21)
|
||||||
|
]
|
||||||
|
create_channel_db(TEST_DATA, "default", channel_id, messages)
|
||||||
|
load_messages = self._import_load_messages()
|
||||||
|
try:
|
||||||
|
result = load_messages("default", channel_id, limit=3, before_message_id=15, search="hello")
|
||||||
|
finally:
|
||||||
|
self._restore()
|
||||||
|
# IDs matching "hello-" below 15: 13, 11, 9, 7, 5, 3, 1 → LIMIT 3 → [9, 11, 13] (ascending)
|
||||||
|
ids = [m["message_id"] for m in result]
|
||||||
|
assert ids == [9, 11, 13], f"got {ids}"
|
||||||
|
assert all("hello" in m["text"] for m in result)
|
||||||
|
|
||||||
|
def test_empty_result_no_error(self):
|
||||||
|
channel_id = "-100emptysearch"
|
||||||
|
messages = [{"message_id": 1, "date": "2024-01-01", "message": "only one"}]
|
||||||
|
create_channel_db(TEST_DATA, "default", channel_id, messages)
|
||||||
|
load_messages = self._import_load_messages()
|
||||||
|
try:
|
||||||
|
result = load_messages("default", channel_id, limit=100, search="nonexistent")
|
||||||
|
finally:
|
||||||
|
self._restore()
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestHealthPayload:
|
||||||
|
"""Health endpoint structure."""
|
||||||
|
|
||||||
|
def test_health_payload_structure(self):
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(TEST_DATA, aid)
|
||||||
|
payload = health_payload(
|
||||||
|
TEST_DATA,
|
||||||
|
TEST_SESSION,
|
||||||
|
get_global_store(TEST_DATA),
|
||||||
|
{},
|
||||||
|
0,
|
||||||
|
[aid],
|
||||||
|
)
|
||||||
|
assert "ok" in payload
|
||||||
|
assert "status" in payload
|
||||||
|
assert "checks" in payload
|
||||||
|
checks = payload["checks"]
|
||||||
|
assert "data_dir" in checks
|
||||||
|
assert "session_dir" in checks
|
||||||
|
assert "accounts" in checks
|
||||||
|
assert "items" in checks["accounts"]
|
||||||
|
assert aid in checks["accounts"]["items"]
|
||||||
|
|
||||||
|
def test_health_accounts_wrapper(self):
|
||||||
|
a1, a2 = make_account_id(), make_account_id()
|
||||||
|
create_account(TEST_DATA, a1)
|
||||||
|
create_account(TEST_DATA, a2)
|
||||||
|
payload = health_payload(
|
||||||
|
TEST_DATA, TEST_SESSION,
|
||||||
|
get_global_store(TEST_DATA),
|
||||||
|
{}, 0, [a1, a2],
|
||||||
|
)
|
||||||
|
acc_checks = payload["checks"]["accounts"]
|
||||||
|
assert acc_checks["ok"] is True
|
||||||
|
assert len(acc_checks["items"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
class TestPersistence:
|
||||||
|
"""UI persistence helpers."""
|
||||||
|
|
||||||
|
def test_active_account_localstorage(self):
|
||||||
|
"""Verifies the save/load logic pattern used in app.js."""
|
||||||
|
accounts = [
|
||||||
|
{"id": "work", "label": "Work"},
|
||||||
|
{"id": "home", "label": "Home"},
|
||||||
|
]
|
||||||
|
saved_id = "work"
|
||||||
|
loaded = None
|
||||||
|
if saved_id and any(a["id"] == saved_id for a in accounts):
|
||||||
|
loaded = saved_id
|
||||||
|
if not loaded:
|
||||||
|
loaded = next((a["id"] for a in accounts), None)
|
||||||
|
assert loaded == "work"
|
||||||
|
|
||||||
|
saved_id = "longgone"
|
||||||
|
loaded = None
|
||||||
|
if saved_id and any(a["id"] == saved_id for a in accounts):
|
||||||
|
loaded = saved_id
|
||||||
|
if not loaded and accounts:
|
||||||
|
loaded = accounts[0]["id"]
|
||||||
|
assert loaded == "work"
|
||||||
|
|
||||||
|
def test_viewer_fallback_on_missing_account(self):
|
||||||
|
"""Viewer's loadViewerAccount fallback logic."""
|
||||||
|
accounts = [
|
||||||
|
{"id": "alpha", "label": "Alpha"},
|
||||||
|
{"id": "beta", "label": "Beta"},
|
||||||
|
]
|
||||||
|
requested = "nonexistent"
|
||||||
|
requested_exists = requested and any(a["id"] == requested for a in accounts)
|
||||||
|
account_id = requested if requested_exists else (accounts[0]["id"] if accounts else None)
|
||||||
|
assert account_id == "alpha"
|
||||||
|
|
||||||
|
account_id2 = requested if (requested and any(a["id"] == requested for a in [])) else None
|
||||||
|
assert account_id2 is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestJobDeduplicationSchema:
|
||||||
|
"""JobRunner create_job dedup logic."""
|
||||||
|
|
||||||
|
def _import_job_runner(self):
|
||||||
|
import webui_server as ws_module
|
||||||
|
return ws_module.JobRunner
|
||||||
|
|
||||||
|
def test_create_job_dedup_by_account(self):
|
||||||
|
JobRunner = self._import_job_runner()
|
||||||
|
runner = JobRunner()
|
||||||
|
runner.queue.put = lambda job: None
|
||||||
|
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
|
||||||
|
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
|
||||||
|
assert j1.job_id == j2.job_id, "duplicate job was created instead of being reused"
|
||||||
|
assert "Reused existing active job" in j2.logs
|
||||||
|
runner.shutdown(timeout=0)
|
||||||
|
|
||||||
|
def test_create_job_allows_different_accounts(self):
|
||||||
|
JobRunner = self._import_job_runner()
|
||||||
|
runner = JobRunner()
|
||||||
|
runner.queue.put = lambda job: None
|
||||||
|
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
|
||||||
|
time.sleep(0.002) # ensure different timestamp -> different job_id
|
||||||
|
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc2"})
|
||||||
|
assert j1.job_id != j2.job_id
|
||||||
|
runner.shutdown(timeout=0)
|
||||||
|
|
||||||
|
def test_create_job_allows_after_previous_completes(self):
|
||||||
|
JobRunner = self._import_job_runner()
|
||||||
|
runner = JobRunner()
|
||||||
|
runner.queue.put = lambda job: None
|
||||||
|
j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
|
||||||
|
j1.status = "done"
|
||||||
|
time.sleep(0.002) # ensure different timestamp -> different job_id
|
||||||
|
j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
|
||||||
|
assert j1.job_id != j2.job_id
|
||||||
|
runner.shutdown(timeout=0)
|
||||||
|
|
||||||
|
|
||||||
|
class TestAccountHealthSummary:
|
||||||
|
"""account_health_summary structure."""
|
||||||
|
|
||||||
|
def _import_health_summary(self):
|
||||||
|
import webui_server as ws_module
|
||||||
|
self._ws_orig_data = ws_module.DATA_DIR
|
||||||
|
self._ws_orig_session = ws_module.SESSION_DIR
|
||||||
|
ws_module.DATA_DIR = TEST_DATA
|
||||||
|
ws_module.SESSION_DIR = TEST_SESSION
|
||||||
|
return ws_module.account_health_summary, ws_module.JobRunner, ws_module
|
||||||
|
|
||||||
|
def test_health_summary_structure(self):
|
||||||
|
account_health_summary, JobRunner, ws = self._import_health_summary()
|
||||||
|
aid = make_account_id()
|
||||||
|
create_account(TEST_DATA, aid, api_id=1, api_hash="x")
|
||||||
|
try:
|
||||||
|
runner = JobRunner()
|
||||||
|
health = account_health_summary(aid, runner)
|
||||||
|
finally:
|
||||||
|
ws.DATA_DIR = self._ws_orig_data
|
||||||
|
ws.SESSION_DIR = self._ws_orig_session
|
||||||
|
|
||||||
|
assert health["account_id"] == aid
|
||||||
|
assert "label" in health
|
||||||
|
assert "data_dir_exists" in health
|
||||||
|
assert "session_ready" in health
|
||||||
|
assert "api_credentials" in health
|
||||||
|
assert health["api_credentials"] is True
|
||||||
|
assert "channel_count" in health
|
||||||
|
assert "message_count" in health
|
||||||
|
assert "media_count" in health
|
||||||
|
|
||||||
|
|
||||||
|
# ── Cleanup all temp data ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_test_data():
|
||||||
|
if TEST_TMP.exists():
|
||||||
|
shutil.rmtree(str(TEST_TMP), ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
import atexit # noqa: E402
|
||||||
|
atexit.register(cleanup_test_data) # noqa: E402
|
||||||
+1046
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,377 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>Telegram Scraper Control Panel</title>
|
||||||
|
<link rel="stylesheet" href="/static/style.css?v=8" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="app-shell">
|
||||||
|
<aside class="sidebar">
|
||||||
|
<div class="brand-block">
|
||||||
|
<svg class="brand-icon" viewBox="0 0 24 24" aria-hidden="true">
|
||||||
|
<path d="m21 3-7.6 18-3.8-7.6L2 9.6zM9.6 13.4 14 10" />
|
||||||
|
</svg>
|
||||||
|
<div class="eyebrow">Telegram Scraper</div>
|
||||||
|
<h1>Control</h1>
|
||||||
|
<p class="muted">Local scraper console for channels, jobs, and continuous runs.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<nav class="nav-links">
|
||||||
|
<a class="nav-link active" href="/"
|
||||||
|
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true">
|
||||||
|
<rect x="4" y="4" width="6" height="6" />
|
||||||
|
<rect x="14" y="4" width="6" height="6" />
|
||||||
|
<rect x="4" y="14" width="6" height="6" />
|
||||||
|
<rect x="14" y="14" width="6" height="6" /></svg
|
||||||
|
>Dashboard</a
|
||||||
|
>
|
||||||
|
<a class="nav-link" href="/settings"
|
||||||
|
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true">
|
||||||
|
<circle cx="12" cy="12" r="3" />
|
||||||
|
<path
|
||||||
|
d="M12 2v3m0 14v3M2 12h3m14 0h3m-2.9-7.1-2.1 2.1M4.9 19.1 7 17m0-10-2.1-2.1m12.2 14.2-2.1-2.1" /></svg
|
||||||
|
>Settings</a
|
||||||
|
>
|
||||||
|
<a class="nav-link" href="/viewer"
|
||||||
|
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true"><path d="M4 5h16v12H8l-4 3z" /></svg>Message
|
||||||
|
Viewer</a
|
||||||
|
>
|
||||||
|
<a class="nav-link" href="/swagger"
|
||||||
|
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true">
|
||||||
|
<path d="m8 5-5 7 5 7m8-14 5 7-5 7M14 3l-4 18" /></svg
|
||||||
|
>API Docs</a
|
||||||
|
>
|
||||||
|
<a class="nav-link" href="/health"
|
||||||
|
><svg class="nav-icon" viewBox="0 0 24 24" aria-hidden="true"><path d="M3 12h4l2-6 4 12 2-6h6" /></svg
|
||||||
|
>Health</a
|
||||||
|
>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<section class="status-card" id="active-account-card">
|
||||||
|
<div class="section-title">Active Account</div>
|
||||||
|
<div id="active-account-name" class="status-badge">Loading...</div>
|
||||||
|
<p id="active-account-status" class="muted small"></p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section class="status-card">
|
||||||
|
<div class="section-title">Actions</div>
|
||||||
|
<button class="button primary" id="scrape-all-btn">Scrape all channels</button>
|
||||||
|
<button class="button" id="export-all-btn">Export all data</button>
|
||||||
|
<button class="button" id="refresh-dialogs-btn">Refresh dialogs</button>
|
||||||
|
<a class="button" id="open-settings-btn" href="/settings">Settings</a>
|
||||||
|
</section>
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
<main class="content">
|
||||||
|
<header class="dashboard-topbar">
|
||||||
|
<div class="dashboard-title"><span class="menu-icon" aria-hidden="true">☰</span>Dashboard</div>
|
||||||
|
<div class="runtime-status">
|
||||||
|
<span class="runtime-dot"></span>Service: Running <i></i> Scraper: Idle <i></i
|
||||||
|
><span class="runtime-clock"></span>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<!-- Account Tabs -->
|
||||||
|
<div id="account-tabs" class="account-tabs"></div>
|
||||||
|
|
||||||
|
<!-- Account Dashboard Panels -->
|
||||||
|
<div id="account-panels">
|
||||||
|
<!-- Each account's dashboard is rendered here dynamically -->
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Fallback for no accounts -->
|
||||||
|
<section id="no-accounts-msg" class="panel hidden">
|
||||||
|
<div class="panel-header">
|
||||||
|
<div>
|
||||||
|
<h2>No Accounts Configured</h2>
|
||||||
|
<p class="muted">
|
||||||
|
Click "Add Account" in Settings to get started, or use the legacy single-account mode.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- ════════ Settings Dialog ════════ -->
|
||||||
|
<dialog id="settings-dialog" class="settings-dialog">
|
||||||
|
<div class="dialog-shell">
|
||||||
|
<div class="dialog-header">
|
||||||
|
<div>
|
||||||
|
<div class="eyebrow">Settings</div>
|
||||||
|
<h2>Global & Accounts</h2>
|
||||||
|
</div>
|
||||||
|
<button class="button button-small" id="close-settings-btn" type="button">Close</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- ── Accounts Management ── -->
|
||||||
|
<section class="settings-section">
|
||||||
|
<div class="section-title">Accounts</div>
|
||||||
|
<div id="accounts-list" class="accounts-list"></div>
|
||||||
|
|
||||||
|
<form id="add-account-form" class="stack-form add-account-form">
|
||||||
|
<input id="add-account-id" name="account_id" placeholder="Account ID (e.g. work)" required />
|
||||||
|
<input id="add-account-label" name="label" placeholder="Display name (e.g. Work Account)" />
|
||||||
|
<input id="add-account-api-id" name="api_id" placeholder="API ID" />
|
||||||
|
<input id="add-account-api-hash" name="api_hash" placeholder="API Hash" />
|
||||||
|
<button class="button primary" type="submit">Add Account</button>
|
||||||
|
</form>
|
||||||
|
<label class="import-account-row">
|
||||||
|
<span class="muted small">Import account settings JSON</span>
|
||||||
|
<input id="import-account-file" type="file" accept="application/json,.json" />
|
||||||
|
</label>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<!-- ── Account Auth ── (shown per account selected in UI) -->
|
||||||
|
<section class="settings-section" id="account-auth-section">
|
||||||
|
<div class="section-title">Account Auth: <span id="auth-account-label">-</span></div>
|
||||||
|
<form id="credentials-form" class="stack-form">
|
||||||
|
<input id="api-id-input" name="api_id" placeholder="API ID" />
|
||||||
|
<input id="api-hash-input" name="api_hash" placeholder="API Hash" />
|
||||||
|
<button class="button" type="submit">Save credentials</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<div class="auth-actions">
|
||||||
|
<button class="button primary" id="start-qr-btn" type="button">Start QR login</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="qr-wrap" class="qr-wrap hidden">
|
||||||
|
<img id="qr-image" alt="Telegram QR login" />
|
||||||
|
<p class="muted small">Open Telegram -> Settings -> Devices -> Scan QR.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form id="phone-form" class="stack-form">
|
||||||
|
<input id="phone-input" name="phone" placeholder="+1234567890" />
|
||||||
|
<button class="button" type="submit">Send code</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<form id="code-form" class="stack-form hidden">
|
||||||
|
<input id="code-input" name="code" placeholder="Telegram code" />
|
||||||
|
<button class="button" type="submit">Confirm code</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<form id="password-form" class="stack-form hidden">
|
||||||
|
<input id="password-input" name="password" type="password" placeholder="2FA password" />
|
||||||
|
<button class="button" type="submit">Confirm password</button>
|
||||||
|
</form>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<!-- ── Scraping ── -->
|
||||||
|
<section class="settings-section">
|
||||||
|
<div class="section-title">Scraping</div>
|
||||||
|
<label class="toggle-row">
|
||||||
|
<span>Download media</span>
|
||||||
|
<span class="switch">
|
||||||
|
<input id="scrape-media-toggle" type="checkbox" />
|
||||||
|
<span class="switch-slider"></span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
</dialog>
|
||||||
|
|
||||||
|
<!-- ════════ Templates ════════ -->
|
||||||
|
|
||||||
|
<!-- Account Tab -->
|
||||||
|
<template id="account-tab-template">
|
||||||
|
<button class="account-tab" type="button">
|
||||||
|
<span class="account-tab-name"></span>
|
||||||
|
<span class="account-tab-status"></span>
|
||||||
|
</button>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<!-- Account Dashboard Panel -->
|
||||||
|
<template id="account-panel-template">
|
||||||
|
<section class="account-panel">
|
||||||
|
<!-- Dashboard tabs: Overview, Continuous -->
|
||||||
|
<div class="content-tabs">
|
||||||
|
<button class="content-tab active" data-tab-target="overview" type="button">Overview</button>
|
||||||
|
<button class="content-tab" data-tab-target="continuous" type="button">Continuous</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Overview Tab -->
|
||||||
|
<div class="tab-panel active" data-panel="overview">
|
||||||
|
<section class="panel-grid">
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Channels</div>
|
||||||
|
<div class="stat-value channel-count">0</div>
|
||||||
|
<div class="stat-caption">Tracked channels</div>
|
||||||
|
<div class="stat-meta">
|
||||||
|
<span><strong class="channel-active-count">0</strong> Active</span
|
||||||
|
><span><strong class="channel-inactive-count">0</strong> Inactive</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Media scraping</div>
|
||||||
|
<div class="stat-value stat-compact scrape-media-label">OFF</div>
|
||||||
|
<div class="stat-caption">Items downloaded</div>
|
||||||
|
<div class="stat-meta">
|
||||||
|
<span><strong class="media-session-count">0</strong> This session</span
|
||||||
|
><span><strong class="media-total-count">0</strong> Total</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Forwarding rules</div>
|
||||||
|
<div class="stat-value forwarding-count">0</div>
|
||||||
|
<div class="stat-caption">Active rules</div>
|
||||||
|
<div class="stat-meta">
|
||||||
|
<span><strong class="forwarding-active-count">0</strong> Active</span
|
||||||
|
><span><strong>0</strong> Paused</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Account health</div>
|
||||||
|
<div class="stat-value stat-compact account-health-label">-</div>
|
||||||
|
<div class="stat-caption">Session status</div>
|
||||||
|
<div class="stat-meta">
|
||||||
|
<span><strong class="health-session-value">-</strong> Session</span
|
||||||
|
><span><strong class="health-rate-value">-</strong> Rate limit</span>
|
||||||
|
</div>
|
||||||
|
<div class="muted small account-health-detail"></div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section class="panel">
|
||||||
|
<div class="panel-header">
|
||||||
|
<div>
|
||||||
|
<h2>Tracked Channels</h2>
|
||||||
|
<p class="muted">Per-account channel list.</p>
|
||||||
|
</div>
|
||||||
|
<form class="inline-form add-channel-form">
|
||||||
|
<input class="add-channel-id" name="channel_id" placeholder="ID or @username" required />
|
||||||
|
<input class="add-channel-name" name="name" placeholder="Display name" />
|
||||||
|
<button class="button primary" type="submit">Add</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="table-wrap">
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Channel</th>
|
||||||
|
<th>Messages</th>
|
||||||
|
<th>Media</th>
|
||||||
|
<th>Last message</th>
|
||||||
|
<th>Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody class="channels-table"></tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section class="panel jobs-panel">
|
||||||
|
<div class="panel-header">
|
||||||
|
<div>
|
||||||
|
<h2>Jobs</h2>
|
||||||
|
<p class="muted">Queued work runs one job at a time.</p>
|
||||||
|
</div>
|
||||||
|
<button class="button refresh-jobs-btn">Refresh</button>
|
||||||
|
</div>
|
||||||
|
<div class="jobs-list"></div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Continuous Tab -->
|
||||||
|
<div class="tab-panel" data-panel="continuous">
|
||||||
|
<section class="panel">
|
||||||
|
<div class="panel-header">
|
||||||
|
<div>
|
||||||
|
<h2>Continuous Scraping</h2>
|
||||||
|
<p class="muted">Per-account continuous scrape settings.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form class="continuous-form">
|
||||||
|
<label class="toggle-row">
|
||||||
|
<span>Enabled</span>
|
||||||
|
<span class="switch">
|
||||||
|
<input type="checkbox" class="continuous-enabled" checked />
|
||||||
|
<span class="switch-slider"></span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<label class="stack-form">
|
||||||
|
<span class="muted small">Interval, minutes</span>
|
||||||
|
<input type="number" class="continuous-interval" min="1" value="1" />
|
||||||
|
</label>
|
||||||
|
<label class="toggle-row">
|
||||||
|
<span>All tracked channels</span>
|
||||||
|
<span class="switch">
|
||||||
|
<input type="checkbox" class="continuous-all" checked />
|
||||||
|
<span class="switch-slider"></span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<div class="channel-picker">
|
||||||
|
<div class="muted small">Continuous channel set</div>
|
||||||
|
<div class="checkbox-grid continuous-channel-list"></div>
|
||||||
|
</div>
|
||||||
|
<button class="button primary" type="submit">Save continuous settings</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<div class="continuous-meta">
|
||||||
|
<div>Status: <span class="continuous-status">-</span></div>
|
||||||
|
<div>Last run: <span class="continuous-last-iteration">-</span></div>
|
||||||
|
<div>Next run: <span class="continuous-next-run">-</span></div>
|
||||||
|
<div>Last error: <span class="continuous-last-error">-</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="log-viewer continuous-logs"></div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<!-- Channel Row -->
|
||||||
|
<template id="channel-row-template">
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<div class="channel-name"></div>
|
||||||
|
<div class="muted small channel-id"></div>
|
||||||
|
</td>
|
||||||
|
<td class="message-count"></td>
|
||||||
|
<td class="media-count"></td>
|
||||||
|
<td class="last-date"></td>
|
||||||
|
<td>
|
||||||
|
<div class="action-row">
|
||||||
|
<button class="button button-small scrape-btn">Scrape</button>
|
||||||
|
<button class="button button-small export-btn">Export</button>
|
||||||
|
<button class="button button-small export-view-btn">Viewer</button>
|
||||||
|
<button class="button button-small media-btn">Rescrape media</button>
|
||||||
|
<button class="button button-small danger remove-btn">Remove</button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<!-- Job Template -->
|
||||||
|
<template id="job-template">
|
||||||
|
<article class="job-card">
|
||||||
|
<div class="job-head">
|
||||||
|
<div class="job-title"></div>
|
||||||
|
<div class="job-status"></div>
|
||||||
|
</div>
|
||||||
|
<div class="muted small job-time"></div>
|
||||||
|
<pre class="job-logs"></pre>
|
||||||
|
</article>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<!-- Account list item in settings -->
|
||||||
|
<template id="account-list-item-template">
|
||||||
|
<div class="account-list-item">
|
||||||
|
<div class="account-list-info">
|
||||||
|
<span class="account-list-label"></span>
|
||||||
|
<span class="muted small account-list-id"></span>
|
||||||
|
<span class="account-list-auth-status"></span>
|
||||||
|
</div>
|
||||||
|
<div class="account-list-actions">
|
||||||
|
<button class="button button-small account-select-btn">Select</button>
|
||||||
|
<button class="button button-small account-export-btn">Export</button>
|
||||||
|
<button class="button button-small danger account-remove-btn">Remove</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script src="/static/app.js?v=4"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>Telegram Scraper Settings</title>
|
||||||
|
<link rel="stylesheet" href="/static/style.css?v=8" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="app-shell settings-page-shell">
|
||||||
|
<aside class="sidebar">
|
||||||
|
<div class="brand-block">
|
||||||
|
<div class="eyebrow">Telegram Scraper</div>
|
||||||
|
<h1>Settings</h1>
|
||||||
|
<p class="muted">Account credentials, import/export, and runtime safety controls.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<nav class="nav-links">
|
||||||
|
<a class="nav-link" href="/">Dashboard</a>
|
||||||
|
<a class="nav-link active" href="/settings">Settings</a>
|
||||||
|
<a class="nav-link" href="/viewer">Message Viewer</a>
|
||||||
|
<a class="nav-link" href="/swagger">API Docs</a>
|
||||||
|
<a class="nav-link" href="/health">Health</a>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<section class="status-card">
|
||||||
|
<div class="section-title">Accounts</div>
|
||||||
|
<div id="settings-account-list" class="settings-account-list"></div>
|
||||||
|
</section>
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
<main class="content settings-page">
|
||||||
|
<section class="panel settings-hero">
|
||||||
|
<div>
|
||||||
|
<div class="eyebrow">Account Settings</div>
|
||||||
|
<h2 id="settings-title">Loading accounts</h2>
|
||||||
|
<p id="settings-subtitle" class="muted">Select an account to manage credentials and scraper options.</p>
|
||||||
|
</div>
|
||||||
|
<div class="action-row">
|
||||||
|
<button class="button" id="settings-export-btn" type="button" disabled>Export</button>
|
||||||
|
<button class="button danger" id="settings-delete-btn" type="button" disabled>Delete</button>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section id="settings-empty" class="panel hidden">
|
||||||
|
<div class="empty-state">
|
||||||
|
<h2>Add first account</h2>
|
||||||
|
<p class="muted">Create an account below or import an exported account JSON.</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section class="panel-grid settings-health-grid">
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Credentials</div>
|
||||||
|
<div class="stat-value stat-compact" id="health-credentials">-</div>
|
||||||
|
</div>
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Session</div>
|
||||||
|
<div class="stat-value stat-compact" id="health-session">-</div>
|
||||||
|
</div>
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Continuous</div>
|
||||||
|
<div class="stat-value stat-compact" id="health-continuous">-</div>
|
||||||
|
</div>
|
||||||
|
<div class="panel stat-panel">
|
||||||
|
<div class="section-title">Last scrape</div>
|
||||||
|
<div class="stat-value stat-compact" id="health-last-scrape">-</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section class="settings-layout">
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-header">
|
||||||
|
<div>
|
||||||
|
<h2>Create or Import</h2>
|
||||||
|
<p class="muted">New accounts start without continuous scraping enabled.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<form id="settings-add-account-form" class="stack-form">
|
||||||
|
<input id="settings-add-account-id" name="account_id" placeholder="Account ID (e.g. work)" required />
|
||||||
|
<input id="settings-add-account-label" name="label" placeholder="Display name" />
|
||||||
|
<input id="settings-add-account-api-id" name="api_id" placeholder="API ID" />
|
||||||
|
<input id="settings-add-account-api-hash" name="api_hash" placeholder="API Hash" />
|
||||||
|
<button class="button primary" type="submit">Add Account</button>
|
||||||
|
</form>
|
||||||
|
<label class="import-account-row">
|
||||||
|
<span class="muted small">Import account settings JSON</span>
|
||||||
|
<input id="settings-import-account-file" type="file" accept="application/json,.json" />
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-header">
|
||||||
|
<div>
|
||||||
|
<h2>Credentials & Auth</h2>
|
||||||
|
<p class="muted" id="settings-auth-label">No account selected.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<form id="settings-credentials-form" class="stack-form">
|
||||||
|
<input id="settings-api-id-input" name="api_id" placeholder="API ID" />
|
||||||
|
<input id="settings-api-hash-input" name="api_hash" placeholder="API Hash" />
|
||||||
|
<button class="button" type="submit">Save credentials</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<div class="auth-actions">
|
||||||
|
<button class="button primary" id="settings-start-qr-btn" type="button">Start QR login</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="settings-qr-wrap" class="qr-wrap hidden">
|
||||||
|
<img id="settings-qr-image" alt="Telegram QR login" />
|
||||||
|
<p class="muted small">Open Telegram -> Settings -> Devices -> Scan QR.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form id="settings-phone-form" class="stack-form">
|
||||||
|
<input id="settings-phone-input" name="phone" placeholder="+1234567890" />
|
||||||
|
<button class="button" type="submit">Send code</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<form id="settings-code-form" class="stack-form hidden">
|
||||||
|
<input id="settings-code-input" name="code" placeholder="Telegram code" />
|
||||||
|
<button class="button" type="submit">Confirm code</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<form id="settings-password-form" class="stack-form hidden">
|
||||||
|
<input id="settings-password-input" name="password" type="password" placeholder="2FA password" />
|
||||||
|
<button class="button" type="submit">Confirm password</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-header">
|
||||||
|
<div>
|
||||||
|
<h2>Scraping</h2>
|
||||||
|
<p class="muted">Account-level parser settings.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<label class="toggle-row">
|
||||||
|
<span>Download media</span>
|
||||||
|
<span class="switch">
|
||||||
|
<input id="settings-scrape-media-toggle" type="checkbox" />
|
||||||
|
<span class="switch-slider"></span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<div class="settings-meta-list">
|
||||||
|
<div>Tracked channels: <span id="settings-channel-count">0</span></div>
|
||||||
|
<div>Messages: <span id="settings-message-count">0</span></div>
|
||||||
|
<div>Media: <span id="settings-media-count">0</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script src="/static/settings.js?v=1"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,431 @@
|
|||||||
|
const settingsState = {
|
||||||
|
accounts: [],
|
||||||
|
activeAccount: null,
|
||||||
|
dashboard: null,
|
||||||
|
auth: null,
|
||||||
|
continuous: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
const ACTIVE_ACCOUNT_STORAGE_KEY = 'telegramScraper.activeAccount';
|
||||||
|
|
||||||
|
async function api(path, options = {}) {
|
||||||
|
const response = await fetch(path, {
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
...options,
|
||||||
|
});
|
||||||
|
const data = await response.json();
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(data.error || 'Request failed');
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
function showToast(message, type = 'info') {
|
||||||
|
let root = document.getElementById('toast-root');
|
||||||
|
if (!root) {
|
||||||
|
root = document.createElement('div');
|
||||||
|
root.id = 'toast-root';
|
||||||
|
root.className = 'toast-root';
|
||||||
|
document.body.appendChild(root);
|
||||||
|
}
|
||||||
|
const toast = document.createElement('div');
|
||||||
|
toast.className = `toast toast-${type}`;
|
||||||
|
toast.textContent = message;
|
||||||
|
root.appendChild(toast);
|
||||||
|
window.setTimeout(() => {
|
||||||
|
toast.classList.add('toast-hide');
|
||||||
|
window.setTimeout(() => toast.remove(), 250);
|
||||||
|
}, 3600);
|
||||||
|
}
|
||||||
|
|
||||||
|
function downloadJson(filename, payload) {
|
||||||
|
const blob = new Blob([JSON.stringify(payload, null, 2) + '\n'], {
|
||||||
|
type: 'application/json',
|
||||||
|
});
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const link = document.createElement('a');
|
||||||
|
link.href = url;
|
||||||
|
link.download = filename;
|
||||||
|
document.body.appendChild(link);
|
||||||
|
link.click();
|
||||||
|
link.remove();
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
function displayTime(value) {
|
||||||
|
if (!value) return '-';
|
||||||
|
const date = new Date(value);
|
||||||
|
if (Number.isNaN(date.getTime())) return String(value).replace('T', ' ').replace('+00:00', ' UTC');
|
||||||
|
return date.toLocaleString(undefined, {
|
||||||
|
year: 'numeric',
|
||||||
|
month: '2-digit',
|
||||||
|
day: '2-digit',
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAccountAuthorized(auth) {
|
||||||
|
const status = auth?.auth_status || auth || {};
|
||||||
|
return (
|
||||||
|
auth?.phase === 'authorized' ||
|
||||||
|
auth?.status === 'authorized' ||
|
||||||
|
status.status === 'ready' ||
|
||||||
|
status.status === 'authorized'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function accountLabel(account) {
|
||||||
|
return account?.label || account?.id || 'No account';
|
||||||
|
}
|
||||||
|
|
||||||
|
function setActiveAccount(accountId) {
|
||||||
|
settingsState.activeAccount = accountId || null;
|
||||||
|
if (accountId) {
|
||||||
|
localStorage.setItem(ACTIVE_ACCOUNT_STORAGE_KEY, accountId);
|
||||||
|
} else {
|
||||||
|
localStorage.removeItem(ACTIVE_ACCOUNT_STORAGE_KEY);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function chooseInitialAccount() {
|
||||||
|
const requested = new URLSearchParams(window.location.search).get('account');
|
||||||
|
if (requested && settingsState.accounts.some((account) => account.id === requested)) {
|
||||||
|
return requested;
|
||||||
|
}
|
||||||
|
const saved = localStorage.getItem(ACTIVE_ACCOUNT_STORAGE_KEY);
|
||||||
|
if (saved && settingsState.accounts.some((account) => account.id === saved)) {
|
||||||
|
return saved;
|
||||||
|
}
|
||||||
|
return settingsState.accounts[0]?.id || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderAccountList() {
|
||||||
|
const root = document.getElementById('settings-account-list');
|
||||||
|
root.innerHTML = '';
|
||||||
|
|
||||||
|
if (!settingsState.accounts.length) {
|
||||||
|
root.innerHTML = '<div class="empty-state">No accounts yet.</div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
settingsState.accounts.forEach((account) => {
|
||||||
|
const button = document.createElement('button');
|
||||||
|
button.className = 'settings-account-button';
|
||||||
|
button.type = 'button';
|
||||||
|
button.classList.toggle('active', account.id === settingsState.activeAccount);
|
||||||
|
|
||||||
|
const name = document.createElement('span');
|
||||||
|
name.className = 'settings-account-name';
|
||||||
|
name.textContent = accountLabel(account);
|
||||||
|
|
||||||
|
const status = document.createElement('span');
|
||||||
|
status.className = isAccountAuthorized(account.auth) ? 'settings-account-status ok' : 'settings-account-status';
|
||||||
|
status.textContent = isAccountAuthorized(account.auth) ? 'Authorized' : 'Needs auth';
|
||||||
|
|
||||||
|
button.append(name, status);
|
||||||
|
button.addEventListener('click', () => loadAccount(account.id));
|
||||||
|
root.appendChild(button);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderEmptyState() {
|
||||||
|
const hasAccount = Boolean(settingsState.activeAccount);
|
||||||
|
document.getElementById('settings-empty').classList.toggle('hidden', hasAccount);
|
||||||
|
document.getElementById('settings-export-btn').disabled = !hasAccount;
|
||||||
|
document.getElementById('settings-delete-btn').disabled = !hasAccount;
|
||||||
|
document
|
||||||
|
.getElementById('settings-credentials-form')
|
||||||
|
.querySelectorAll('input, button')
|
||||||
|
.forEach((node) => {
|
||||||
|
node.disabled = !hasAccount;
|
||||||
|
});
|
||||||
|
document.getElementById('settings-start-qr-btn').disabled = !hasAccount;
|
||||||
|
document
|
||||||
|
.getElementById('settings-phone-form')
|
||||||
|
.querySelectorAll('input, button')
|
||||||
|
.forEach((node) => {
|
||||||
|
node.disabled = !hasAccount;
|
||||||
|
});
|
||||||
|
document.getElementById('settings-scrape-media-toggle').disabled = !hasAccount;
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderAccountData() {
|
||||||
|
const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount);
|
||||||
|
const dashboard = settingsState.dashboard || {};
|
||||||
|
const health = dashboard.health || {};
|
||||||
|
const auth = settingsState.auth || account?.auth || {};
|
||||||
|
const continuous = settingsState.continuous || {};
|
||||||
|
const continuousStatus = continuous.status || {};
|
||||||
|
const continuousConfig = continuous.config || {};
|
||||||
|
|
||||||
|
document.getElementById('settings-title').textContent = account ? accountLabel(account) : 'No account selected';
|
||||||
|
document.getElementById('settings-subtitle').textContent = account
|
||||||
|
? `Account ID: ${account.id}`
|
||||||
|
: 'Create or import an account to manage settings.';
|
||||||
|
document.getElementById('settings-auth-label').textContent = account
|
||||||
|
? `${accountLabel(account)} auth status`
|
||||||
|
: 'No account selected.';
|
||||||
|
|
||||||
|
document.getElementById('health-credentials').textContent = health.api_credentials ? 'Saved' : 'Missing';
|
||||||
|
document.getElementById('health-credentials').style.color = health.api_credentials ? 'var(--ok)' : 'var(--warn)';
|
||||||
|
document.getElementById('health-session').textContent =
|
||||||
|
health.session_ready || isAccountAuthorized(auth) ? 'Ready' : 'Missing';
|
||||||
|
document.getElementById('health-session').style.color =
|
||||||
|
health.session_ready || isAccountAuthorized(auth) ? 'var(--ok)' : 'var(--warn)';
|
||||||
|
document.getElementById('health-continuous').textContent = continuousStatus.running
|
||||||
|
? 'Running'
|
||||||
|
: continuousConfig.enabled
|
||||||
|
? 'Enabled'
|
||||||
|
: 'Stopped';
|
||||||
|
document.getElementById('health-continuous').style.color = continuousStatus.running ? 'var(--ok)' : 'var(--dim)';
|
||||||
|
document.getElementById('health-last-scrape').textContent = displayTime(
|
||||||
|
health.last_scrape || continuousStatus.last_iteration_at,
|
||||||
|
);
|
||||||
|
|
||||||
|
document.getElementById('settings-scrape-media-toggle').checked = Boolean(dashboard.dashboard?.scrape_media);
|
||||||
|
document.getElementById('settings-channel-count').textContent = String(health.channel_count || 0);
|
||||||
|
document.getElementById('settings-message-count').textContent = String(health.message_count || 0);
|
||||||
|
document.getElementById('settings-media-count').textContent = String(health.media_count || 0);
|
||||||
|
renderEmptyState();
|
||||||
|
renderAccountList();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadAccounts() {
|
||||||
|
const payload = await api('/api/accounts');
|
||||||
|
settingsState.accounts = payload.accounts || [];
|
||||||
|
setActiveAccount(chooseInitialAccount());
|
||||||
|
renderAccountList();
|
||||||
|
if (settingsState.activeAccount) {
|
||||||
|
await loadAccount(settingsState.activeAccount, { preserveUrl: true });
|
||||||
|
} else {
|
||||||
|
settingsState.dashboard = null;
|
||||||
|
settingsState.auth = null;
|
||||||
|
settingsState.continuous = null;
|
||||||
|
renderAccountData();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadAccount(accountId, options = {}) {
|
||||||
|
setActiveAccount(accountId);
|
||||||
|
const [dashboard, auth, continuous] = await Promise.all([
|
||||||
|
api(`/api/accounts/${encodeURIComponent(accountId)}`),
|
||||||
|
api(`/api/accounts/${encodeURIComponent(accountId)}/auth`).catch(() => ({})),
|
||||||
|
api(`/api/accounts/${encodeURIComponent(accountId)}/continuous`).catch(() => ({})),
|
||||||
|
]);
|
||||||
|
settingsState.dashboard = dashboard;
|
||||||
|
settingsState.auth = auth;
|
||||||
|
settingsState.continuous = continuous;
|
||||||
|
if (!options.preserveUrl) {
|
||||||
|
history.replaceState(null, '', `/settings?account=${encodeURIComponent(accountId)}`);
|
||||||
|
}
|
||||||
|
renderAccountData();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createAccount(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
const accountId = document.getElementById('settings-add-account-id').value.trim();
|
||||||
|
const label = document.getElementById('settings-add-account-label').value.trim();
|
||||||
|
const apiId = document.getElementById('settings-add-account-api-id').value.trim();
|
||||||
|
const apiHash = document.getElementById('settings-add-account-api-hash').value.trim();
|
||||||
|
if (!accountId) return;
|
||||||
|
|
||||||
|
await api('/api/accounts', {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({
|
||||||
|
account_id: accountId,
|
||||||
|
label: label || accountId,
|
||||||
|
api_id: apiId ? parseInt(apiId) : undefined,
|
||||||
|
api_hash: apiHash || undefined,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
event.currentTarget.reset();
|
||||||
|
await loadAccounts();
|
||||||
|
await loadAccount(accountId);
|
||||||
|
showToast(`Added ${accountId}.`, 'success');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function importAccount(event) {
|
||||||
|
const file = event.currentTarget.files?.[0];
|
||||||
|
if (!file) return;
|
||||||
|
try {
|
||||||
|
const payload = JSON.parse(await file.text());
|
||||||
|
const accountId = payload.account_id || payload.id;
|
||||||
|
if (!accountId) throw new Error('account_id is missing in import file');
|
||||||
|
await api('/api/accounts/import', {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
event.currentTarget.value = '';
|
||||||
|
await loadAccounts();
|
||||||
|
await loadAccount(accountId);
|
||||||
|
showToast(`Imported ${accountId}.`, 'success');
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Failed to import account: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveCredentials(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
const apiId = document.getElementById('settings-api-id-input').value.trim();
|
||||||
|
const apiHash = document.getElementById('settings-api-hash-input').value.trim();
|
||||||
|
if (!apiId || !apiHash) return;
|
||||||
|
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/credentials`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({ api_id: apiId, api_hash: apiHash }),
|
||||||
|
});
|
||||||
|
await loadAccount(settingsState.activeAccount);
|
||||||
|
showToast('Credentials saved.', 'success');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startQrLogin() {
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
const data = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/qr/start`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({}),
|
||||||
|
});
|
||||||
|
if (data.qr_image) {
|
||||||
|
document.getElementById('settings-qr-image').src = data.qr_image;
|
||||||
|
document.getElementById('settings-qr-wrap').classList.remove('hidden');
|
||||||
|
}
|
||||||
|
await loadAccount(settingsState.activeAccount);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestPhoneCode(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
const phone = document.getElementById('settings-phone-input').value.trim();
|
||||||
|
if (!phone) return;
|
||||||
|
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/request`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({ phone }),
|
||||||
|
});
|
||||||
|
document.getElementById('settings-code-form').classList.remove('hidden');
|
||||||
|
await loadAccount(settingsState.activeAccount);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function submitPhoneCode(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
const code = document.getElementById('settings-code-input').value.trim();
|
||||||
|
if (!code) return;
|
||||||
|
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/phone/submit`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({ code }),
|
||||||
|
});
|
||||||
|
document.getElementById('settings-code-input').value = '';
|
||||||
|
await loadAccount(settingsState.activeAccount);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function submitPassword(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
const password = document.getElementById('settings-password-input').value.trim();
|
||||||
|
if (!password) return;
|
||||||
|
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/auth/password`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({ password }),
|
||||||
|
});
|
||||||
|
document.getElementById('settings-password-input').value = '';
|
||||||
|
await loadAccount(settingsState.activeAccount);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function exportAccount() {
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
const payload = await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/export`);
|
||||||
|
downloadJson(`telegram-scraper-account-${settingsState.activeAccount}.json`, payload);
|
||||||
|
showToast('Account exported.', 'success');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteAccount() {
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
const account = settingsState.accounts.find((item) => item.id === settingsState.activeAccount);
|
||||||
|
if (!window.confirm(`Remove account "${accountLabel(account)}"? All its account data will be deleted.`)) return;
|
||||||
|
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}`, { method: 'DELETE' });
|
||||||
|
setActiveAccount(null);
|
||||||
|
await loadAccounts();
|
||||||
|
showToast('Account deleted.', 'success');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function toggleMedia(event) {
|
||||||
|
if (!settingsState.activeAccount) return;
|
||||||
|
await api(`/api/accounts/${encodeURIComponent(settingsState.activeAccount)}/settings/media`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({ value: event.currentTarget.checked }),
|
||||||
|
});
|
||||||
|
await loadAccount(settingsState.activeAccount);
|
||||||
|
}
|
||||||
|
|
||||||
|
function bindEvents() {
|
||||||
|
document.getElementById('settings-add-account-form').addEventListener('submit', async (event) => {
|
||||||
|
try {
|
||||||
|
await createAccount(event);
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Failed to add account: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-import-account-file').addEventListener('change', importAccount);
|
||||||
|
document.getElementById('settings-credentials-form').addEventListener('submit', async (event) => {
|
||||||
|
try {
|
||||||
|
await saveCredentials(event);
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Failed to save credentials: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-start-qr-btn').addEventListener('click', async () => {
|
||||||
|
try {
|
||||||
|
await startQrLogin();
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`QR login failed: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-phone-form').addEventListener('submit', async (event) => {
|
||||||
|
try {
|
||||||
|
await requestPhoneCode(event);
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Phone login failed: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-code-form').addEventListener('submit', async (event) => {
|
||||||
|
try {
|
||||||
|
await submitPhoneCode(event);
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Code submit failed: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-password-form').addEventListener('submit', async (event) => {
|
||||||
|
try {
|
||||||
|
await submitPassword(event);
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Password submit failed: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-export-btn').addEventListener('click', async () => {
|
||||||
|
try {
|
||||||
|
await exportAccount();
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Export failed: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-delete-btn').addEventListener('click', async () => {
|
||||||
|
try {
|
||||||
|
await deleteAccount();
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Delete failed: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.getElementById('settings-scrape-media-toggle').addEventListener('change', async (event) => {
|
||||||
|
try {
|
||||||
|
await toggleMedia(event);
|
||||||
|
} catch (err) {
|
||||||
|
showToast(`Media setting failed: ${err.message}`, 'error');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
bindEvents();
|
||||||
|
loadAccounts().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
showToast(`Failed to load settings: ${err.message}`, 'error');
|
||||||
|
});
|
||||||
+2224
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>Telegram Scraper API</title>
|
||||||
|
<link rel="stylesheet" href="/static/style.css?v=6" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="viewer-shell">
|
||||||
|
<aside class="viewer-sidebar">
|
||||||
|
<div class="viewer-sidebar-head">
|
||||||
|
<div>
|
||||||
|
<div class="eyebrow">OpenAPI</div>
|
||||||
|
<h1>API Docs</h1>
|
||||||
|
</div>
|
||||||
|
<a class="button button-small" href="/">Dashboard</a>
|
||||||
|
</div>
|
||||||
|
<nav class="nav-links">
|
||||||
|
<a class="nav-link" href="/openapi.json">openapi.json</a>
|
||||||
|
<a class="nav-link" href="/health">Health</a>
|
||||||
|
<a class="nav-link" href="/viewer">Message Viewer</a>
|
||||||
|
</nav>
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
<main class="viewer-main">
|
||||||
|
<header class="viewer-header">
|
||||||
|
<div>
|
||||||
|
<h2>Endpoints</h2>
|
||||||
|
<p class="muted">Local API surface exposed by the web UI server.</p>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section id="api-docs" class="api-docs"></section>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<template id="api-section-template">
|
||||||
|
<section class="panel api-section">
|
||||||
|
<h2 class="api-path"></h2>
|
||||||
|
<div class="api-methods"></div>
|
||||||
|
</section>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<template id="api-method-template">
|
||||||
|
<article class="api-method">
|
||||||
|
<div class="api-method-head">
|
||||||
|
<span class="api-verb"></span>
|
||||||
|
<span class="api-summary"></span>
|
||||||
|
</div>
|
||||||
|
<p class="muted api-description"></p>
|
||||||
|
<pre class="job-logs api-body"></pre>
|
||||||
|
</article>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script src="/static/swagger.js"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
async function loadSpec() {
|
||||||
|
const response = await fetch('/openapi.json');
|
||||||
|
const spec = await response.json();
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(spec.error || 'Failed to load OpenAPI spec');
|
||||||
|
}
|
||||||
|
return spec;
|
||||||
|
}
|
||||||
|
|
||||||
|
function methodPayload(operation) {
|
||||||
|
const body = operation.requestBody?.content?.['application/json']?.schema;
|
||||||
|
if (!body) return '';
|
||||||
|
return JSON.stringify(body.example || body.properties || body, null, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderSpec(spec) {
|
||||||
|
const root = document.getElementById('api-docs');
|
||||||
|
const sectionTemplate = document.getElementById('api-section-template');
|
||||||
|
const methodTemplate = document.getElementById('api-method-template');
|
||||||
|
root.innerHTML = '';
|
||||||
|
|
||||||
|
Object.entries(spec.paths || {}).forEach(([path, methods]) => {
|
||||||
|
const section = sectionTemplate.content.firstElementChild.cloneNode(true);
|
||||||
|
section.querySelector('.api-path').textContent = path;
|
||||||
|
const methodsRoot = section.querySelector('.api-methods');
|
||||||
|
|
||||||
|
Object.entries(methods).forEach(([method, operation]) => {
|
||||||
|
const node = methodTemplate.content.firstElementChild.cloneNode(true);
|
||||||
|
node.querySelector('.api-verb').textContent = method.toUpperCase();
|
||||||
|
node.querySelector('.api-summary').textContent = operation.summary || '';
|
||||||
|
node.querySelector('.api-description').textContent = operation.description || '';
|
||||||
|
|
||||||
|
const body = methodPayload(operation);
|
||||||
|
const bodyNode = node.querySelector('.api-body');
|
||||||
|
if (body) {
|
||||||
|
bodyNode.textContent = body;
|
||||||
|
} else {
|
||||||
|
bodyNode.remove();
|
||||||
|
}
|
||||||
|
|
||||||
|
methodsRoot.appendChild(node);
|
||||||
|
});
|
||||||
|
|
||||||
|
root.appendChild(section);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
loadSpec()
|
||||||
|
.then(renderSpec)
|
||||||
|
.catch((error) => {
|
||||||
|
console.error(error);
|
||||||
|
document.getElementById('api-docs').innerHTML = `<section class="panel">${error.message}</section>`;
|
||||||
|
});
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>Telegram Scraper Viewer</title>
|
||||||
|
<link rel="stylesheet" href="/static/style.css?v=8" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="viewer-shell">
|
||||||
|
<aside class="viewer-sidebar">
|
||||||
|
<div class="viewer-sidebar-head">
|
||||||
|
<div>
|
||||||
|
<div class="eyebrow">Export Viewer</div>
|
||||||
|
<h1>Messages</h1>
|
||||||
|
<p class="muted small">
|
||||||
|
Account:
|
||||||
|
<select id="viewer-account-select"></select>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<button id="sidebar-toggle" class="button button-small sidebar-toggle">☰</button>
|
||||||
|
<a class="button button-small" href="/">Dashboard</a>
|
||||||
|
</div>
|
||||||
|
<div id="viewer-channel-list" class="viewer-channel-list"></div>
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
<main class="viewer-main">
|
||||||
|
<header class="viewer-header">
|
||||||
|
<div>
|
||||||
|
<h2 id="viewer-title">Select a channel</h2>
|
||||||
|
<p id="viewer-subtitle" class="muted">Reading messages from the local SQLite database.</p>
|
||||||
|
</div>
|
||||||
|
<div class="viewer-tools">
|
||||||
|
<input id="viewer-search" type="search" placeholder="Search messages" />
|
||||||
|
<label class="viewer-auto-refresh">
|
||||||
|
<input id="viewer-auto-refresh" type="checkbox" checked />
|
||||||
|
<span>Auto-refresh</span>
|
||||||
|
</label>
|
||||||
|
<button id="viewer-refresh-btn" class="button button-small" type="button">Refresh</button>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section id="messages-list" class="messages-list">
|
||||||
|
<div id="scroll-sentinel" class="scroll-sentinel"></div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<template id="viewer-channel-template">
|
||||||
|
<button class="viewer-channel-item">
|
||||||
|
<span class="viewer-channel-avatar"></span>
|
||||||
|
<span class="viewer-channel-copy">
|
||||||
|
<span class="viewer-channel-row">
|
||||||
|
<span class="viewer-channel-name"></span>
|
||||||
|
<span class="viewer-channel-time"></span>
|
||||||
|
</span>
|
||||||
|
<span class="viewer-channel-row">
|
||||||
|
<span class="viewer-channel-preview"></span>
|
||||||
|
<span class="viewer-channel-count"></span>
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</button>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<template id="message-template">
|
||||||
|
<div class="chat-bubble-wrap">
|
||||||
|
<div class="chat-bubble">
|
||||||
|
<div class="chat-sender"></div>
|
||||||
|
<div class="chat-reply">
|
||||||
|
<div class="chat-reply-border"></div>
|
||||||
|
<div class="chat-reply-body">
|
||||||
|
<div class="chat-reply-author"></div>
|
||||||
|
<div class="chat-reply-text"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="chat-text"></div>
|
||||||
|
<div class="chat-media"></div>
|
||||||
|
<div class="chat-footer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script src="/static/viewer.js"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
+528
@@ -0,0 +1,528 @@
|
|||||||
|
if ('scrollRestoration' in history) history.scrollRestoration = 'manual';
|
||||||
|
|
||||||
|
const viewerState = {
|
||||||
|
accountId: null,
|
||||||
|
accounts: [],
|
||||||
|
channels: [],
|
||||||
|
channelId: null,
|
||||||
|
oldestMessageId: null,
|
||||||
|
newestMessageId: null,
|
||||||
|
userId: null,
|
||||||
|
search: '',
|
||||||
|
loading: false,
|
||||||
|
autoRefreshTimer: null,
|
||||||
|
sentinelObserver: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
async function api(path) {
|
||||||
|
const response = await fetch(path);
|
||||||
|
const data = await response.json();
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(data.error || 'Request failed');
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
function showToast(message, type = 'info') {
|
||||||
|
let root = document.getElementById('toast-root');
|
||||||
|
if (!root) {
|
||||||
|
root = document.createElement('div');
|
||||||
|
root.id = 'toast-root';
|
||||||
|
root.className = 'toast-root';
|
||||||
|
document.body.appendChild(root);
|
||||||
|
}
|
||||||
|
const toast = document.createElement('div');
|
||||||
|
toast.className = `toast toast-${type}`;
|
||||||
|
toast.textContent = message;
|
||||||
|
root.appendChild(toast);
|
||||||
|
window.setTimeout(() => {
|
||||||
|
toast.classList.add('toast-hide');
|
||||||
|
window.setTimeout(() => toast.remove(), 250);
|
||||||
|
}, 3600);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDateHeader(dateStr) {
|
||||||
|
if (!dateStr) return '';
|
||||||
|
const d = new Date(dateStr.replace(' ', 'T'));
|
||||||
|
if (isNaN(d.getTime())) return '';
|
||||||
|
const now = new Date();
|
||||||
|
const diff = now - d;
|
||||||
|
const oneDay = 86400000;
|
||||||
|
if (diff < oneDay && d.getDate() === now.getDate()) return 'Today';
|
||||||
|
if (diff < 2 * oneDay && d.getDate() === now.getDate() - 1) return 'Yesterday';
|
||||||
|
return d.toLocaleDateString('en-US', {
|
||||||
|
month: 'long',
|
||||||
|
day: 'numeric',
|
||||||
|
year: d.getFullYear() !== now.getFullYear() ? 'numeric' : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(dateStr) {
|
||||||
|
if (!dateStr) return '';
|
||||||
|
const d = new Date(dateStr.replace(' ', 'T'));
|
||||||
|
if (isNaN(d.getTime())) return '';
|
||||||
|
return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
|
||||||
|
}
|
||||||
|
|
||||||
|
function chatListTime(dateStr) {
|
||||||
|
if (!dateStr) return '';
|
||||||
|
const d = new Date(dateStr.replace(' ', 'T'));
|
||||||
|
if (isNaN(d.getTime())) return '';
|
||||||
|
const now = new Date();
|
||||||
|
if (d.toDateString() === now.toDateString()) {
|
||||||
|
return d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
|
||||||
|
}
|
||||||
|
return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
|
||||||
|
}
|
||||||
|
|
||||||
|
function initials(value) {
|
||||||
|
const source = String(value || '?')
|
||||||
|
.replace(/^@/, '')
|
||||||
|
.trim();
|
||||||
|
const words = source.split(/\s+/).filter(Boolean);
|
||||||
|
if (words.length > 1) return (words[0][0] + words[1][0]).toUpperCase();
|
||||||
|
return source.slice(0, 2).toUpperCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function dateKey(dateStr) {
|
||||||
|
if (!dateStr) return '';
|
||||||
|
return dateStr.slice(0, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
function textNodeWithBreaks(text) {
|
||||||
|
const fragment = document.createDocumentFragment();
|
||||||
|
if (!text) return fragment;
|
||||||
|
const parts = text.split('\n');
|
||||||
|
parts.forEach((part, index) => {
|
||||||
|
if (index > 0) fragment.appendChild(document.createElement('br'));
|
||||||
|
fragment.appendChild(document.createTextNode(part));
|
||||||
|
});
|
||||||
|
return fragment;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrollToBottom() {
|
||||||
|
const list = document.getElementById('messages-list');
|
||||||
|
if (!list) return;
|
||||||
|
const go = () => {
|
||||||
|
list.scrollTop = list.scrollHeight;
|
||||||
|
};
|
||||||
|
go();
|
||||||
|
requestAnimationFrame(go);
|
||||||
|
setTimeout(go, 150);
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeScrollSentinel() {
|
||||||
|
const sentinel = document.createElement('div');
|
||||||
|
sentinel.id = 'scroll-sentinel';
|
||||||
|
sentinel.className = 'scroll-sentinel';
|
||||||
|
return sentinel;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetMessageView(title = 'Select a channel', subtitle = 'Reading messages from the local SQLite database.') {
|
||||||
|
const root = document.getElementById('messages-list');
|
||||||
|
if (root) {
|
||||||
|
root.innerHTML = '';
|
||||||
|
root.appendChild(makeScrollSentinel());
|
||||||
|
}
|
||||||
|
document.getElementById('viewer-title').textContent = title;
|
||||||
|
document.getElementById('viewer-subtitle').textContent = subtitle;
|
||||||
|
viewerState.oldestMessageId = null;
|
||||||
|
viewerState.newestMessageId = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderChannelList() {
|
||||||
|
const root = document.getElementById('viewer-channel-list');
|
||||||
|
if (!root) return;
|
||||||
|
root.innerHTML = '';
|
||||||
|
const template = document.getElementById('viewer-channel-template');
|
||||||
|
if (!template) return;
|
||||||
|
viewerState.channels.forEach((channel) => {
|
||||||
|
const node = template.content.firstElementChild.cloneNode(true);
|
||||||
|
const preview =
|
||||||
|
channel.last_message_preview ||
|
||||||
|
(channel.has_database ? 'No text in the last saved message' : 'No local database yet');
|
||||||
|
node.querySelector('.viewer-channel-avatar').textContent = initials(channel.name || channel.channel_id);
|
||||||
|
node.querySelector('.viewer-channel-name').textContent = channel.name;
|
||||||
|
node.querySelector('.viewer-channel-time').textContent = chatListTime(channel.last_date);
|
||||||
|
node.querySelector('.viewer-channel-preview').textContent = preview;
|
||||||
|
node.querySelector('.viewer-channel-count').textContent = String(channel.message_count || 0);
|
||||||
|
if (channel.channel_id === viewerState.channelId) {
|
||||||
|
node.classList.add('active');
|
||||||
|
}
|
||||||
|
node.addEventListener('click', () => {
|
||||||
|
document.querySelector('.viewer-sidebar')?.classList.remove('open');
|
||||||
|
viewerState.oldestMessageId = null;
|
||||||
|
viewerState.newestMessageId = null;
|
||||||
|
loadChannel(channel.channel_id);
|
||||||
|
});
|
||||||
|
root.appendChild(node);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function messageEndpoint(channelId, before = '') {
|
||||||
|
const search = viewerState.search ? `&search=${encodeURIComponent(viewerState.search)}` : '';
|
||||||
|
if (viewerState.accountId) {
|
||||||
|
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
|
||||||
|
}
|
||||||
|
return `/api/channels/${encodeURIComponent(channelId)}/messages?limit=80${before}${search}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function channelsEndpoint() {
|
||||||
|
if (viewerState.accountId) {
|
||||||
|
return `/api/accounts/${encodeURIComponent(viewerState.accountId)}/channels`;
|
||||||
|
}
|
||||||
|
return '/api/channels';
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderAccountSelector() {
|
||||||
|
const sel = document.getElementById('viewer-account-select');
|
||||||
|
if (!sel) return;
|
||||||
|
sel.innerHTML = '';
|
||||||
|
const legacy = viewerState.accounts.length === 0;
|
||||||
|
if (legacy) {
|
||||||
|
const opt = document.createElement('option');
|
||||||
|
opt.value = '';
|
||||||
|
opt.textContent = 'Legacy';
|
||||||
|
sel.appendChild(opt);
|
||||||
|
} else {
|
||||||
|
viewerState.accounts.forEach((acc) => {
|
||||||
|
const opt = document.createElement('option');
|
||||||
|
opt.value = acc.id;
|
||||||
|
opt.textContent = acc.label || acc.id;
|
||||||
|
if (acc.id === viewerState.accountId) opt.selected = true;
|
||||||
|
sel.appendChild(opt);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function switchViewerAccount(accountId) {
|
||||||
|
viewerState.accountId = accountId || null;
|
||||||
|
viewerState.search = '';
|
||||||
|
const searchInput = document.getElementById('viewer-search');
|
||||||
|
if (searchInput) searchInput.value = '';
|
||||||
|
viewerState.accounts.forEach((acc) => {
|
||||||
|
if (acc.id === accountId) {
|
||||||
|
const opts = document.getElementById('viewer-account-select')?.options;
|
||||||
|
if (opts) {
|
||||||
|
for (let i = 0; i < opts.length; i++) {
|
||||||
|
opts[i].selected = opts[i].value === accountId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const authData = viewerState.accountId
|
||||||
|
? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`)
|
||||||
|
: await api('/api/auth');
|
||||||
|
viewerState.userId = authData.user_id || null;
|
||||||
|
} catch (e) {
|
||||||
|
console.warn('Could not fetch user ID:', e);
|
||||||
|
}
|
||||||
|
const channels = await api(channelsEndpoint());
|
||||||
|
viewerState.channels = channels;
|
||||||
|
viewerState.channelId = channels[0]?.channel_id || null;
|
||||||
|
viewerState.oldestMessageId = null;
|
||||||
|
viewerState.newestMessageId = null;
|
||||||
|
renderChannelList();
|
||||||
|
if (viewerState.channelId) {
|
||||||
|
await loadChannel(viewerState.channelId);
|
||||||
|
} else {
|
||||||
|
resetMessageView(
|
||||||
|
'No channels',
|
||||||
|
viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadViewerAccount(params) {
|
||||||
|
const requested = params.get('account');
|
||||||
|
try {
|
||||||
|
const payload = await api('/api/accounts');
|
||||||
|
viewerState.accounts = payload.accounts || [];
|
||||||
|
} catch {
|
||||||
|
viewerState.accounts = [];
|
||||||
|
}
|
||||||
|
const requestedExists = requested && viewerState.accounts.some((acc) => acc.id === requested);
|
||||||
|
viewerState.accountId = requestedExists ? requested : viewerState.accounts[0]?.id || null;
|
||||||
|
|
||||||
|
renderAccountSelector();
|
||||||
|
|
||||||
|
const sel = document.getElementById('viewer-account-select');
|
||||||
|
if (sel) {
|
||||||
|
sel.addEventListener('change', () => switchViewerAccount(sel.value));
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const authData = viewerState.accountId
|
||||||
|
? await api(`/api/accounts/${encodeURIComponent(viewerState.accountId)}/auth`)
|
||||||
|
: await api('/api/auth');
|
||||||
|
viewerState.userId = authData.user_id || null;
|
||||||
|
} catch (e) {
|
||||||
|
console.warn('Could not fetch user ID:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMessageNode(message, root) {
|
||||||
|
const template = document.getElementById('message-template');
|
||||||
|
const wrap = template.content.firstElementChild.cloneNode(true);
|
||||||
|
const bubble = wrap.querySelector('.chat-bubble');
|
||||||
|
|
||||||
|
bubble.dataset.messageId = String(message.message_id);
|
||||||
|
|
||||||
|
const isOwn = viewerState.userId && message.sender_id === viewerState.userId;
|
||||||
|
if (isOwn) {
|
||||||
|
wrap.classList.add('chat-bubble-wrap--own');
|
||||||
|
}
|
||||||
|
|
||||||
|
const sender = bubble.querySelector('.chat-sender');
|
||||||
|
sender.textContent = message.sender_name || '';
|
||||||
|
|
||||||
|
const replyEl = bubble.querySelector('.chat-reply');
|
||||||
|
const replyMsg = message.reply_to_message;
|
||||||
|
if (message.reply_to) {
|
||||||
|
replyEl.classList.remove('hidden');
|
||||||
|
bubble.querySelector('.chat-reply-author').textContent = replyMsg?.sender_name || `#${message.reply_to}`;
|
||||||
|
bubble.querySelector('.chat-reply-text').textContent = replyMsg?.text || '(message not available)';
|
||||||
|
replyEl.addEventListener('click', () => {
|
||||||
|
const target = root.querySelector(`[data-message-id="${message.reply_to}"]`);
|
||||||
|
if (target) {
|
||||||
|
target.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||||
|
target.classList.add('chat-bubble--highlight');
|
||||||
|
setTimeout(() => target.classList.remove('chat-bubble--highlight'), 2000);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
replyEl.classList.add('hidden');
|
||||||
|
}
|
||||||
|
|
||||||
|
bubble.querySelector('.chat-text').appendChild(textNodeWithBreaks(message.text || ''));
|
||||||
|
|
||||||
|
const media = bubble.querySelector('.chat-media');
|
||||||
|
if (message.media_url && message.media_kind === 'image') {
|
||||||
|
const img = document.createElement('img');
|
||||||
|
img.src = message.media_url;
|
||||||
|
img.loading = 'lazy';
|
||||||
|
media.appendChild(img);
|
||||||
|
} else if (message.media_url && message.media_kind === 'video') {
|
||||||
|
const video = document.createElement('video');
|
||||||
|
video.src = message.media_url;
|
||||||
|
video.controls = true;
|
||||||
|
video.preload = 'metadata';
|
||||||
|
media.appendChild(video);
|
||||||
|
} else if (message.media_url) {
|
||||||
|
const link = document.createElement('a');
|
||||||
|
link.href = message.media_url;
|
||||||
|
link.target = '_blank';
|
||||||
|
link.rel = 'noreferrer';
|
||||||
|
link.textContent = 'Open file';
|
||||||
|
media.appendChild(link);
|
||||||
|
}
|
||||||
|
|
||||||
|
const footerParts = [];
|
||||||
|
if (message.date) footerParts.push(formatTime(message.date));
|
||||||
|
if (message.views) footerParts.push(`views: ${message.views}`);
|
||||||
|
if (message.forwards) footerParts.push(`forwards: ${message.forwards}`);
|
||||||
|
if (message.reactions) footerParts.push(`reactions: ${message.reactions}`);
|
||||||
|
bubble.querySelector('.chat-footer').textContent = footerParts.join(' | ');
|
||||||
|
|
||||||
|
return { wrap, dateKey: dateKey(message.date) };
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeDateSep(text) {
|
||||||
|
const el = document.createElement('div');
|
||||||
|
el.className = 'chat-date-sep';
|
||||||
|
el.textContent = text;
|
||||||
|
return el;
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderMessages(payload, append = false) {
|
||||||
|
const root = document.getElementById('messages-list');
|
||||||
|
if (!root) return;
|
||||||
|
let sentinel = document.getElementById('scroll-sentinel');
|
||||||
|
if (!sentinel) sentinel = makeScrollSentinel();
|
||||||
|
|
||||||
|
const channel = payload.channel;
|
||||||
|
document.getElementById('viewer-title').textContent = channel?.name || payload.channel_id;
|
||||||
|
document.getElementById('viewer-subtitle').textContent = channel
|
||||||
|
? `${payload.messages.length} shown${viewerState.search ? ` for "${viewerState.search}"` : ''}, ${channel.message_count} total, latest: ${channel.last_date || '-'}`
|
||||||
|
: 'No local database found for this channel yet.';
|
||||||
|
|
||||||
|
if (!append) {
|
||||||
|
root.innerHTML = '';
|
||||||
|
root.appendChild(sentinel);
|
||||||
|
}
|
||||||
|
|
||||||
|
let lastKey = null;
|
||||||
|
if (append) {
|
||||||
|
const firstExisting = root.querySelector('[data-message-id]');
|
||||||
|
if (firstExisting) {
|
||||||
|
const candidate = firstExisting.closest('.chat-bubble-wrap');
|
||||||
|
if (candidate && candidate._dateKey) lastKey = candidate._dateKey;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const items = payload.messages.map((m) => {
|
||||||
|
const { wrap, dateKey: dk } = buildMessageNode(m, root);
|
||||||
|
wrap._dateKey = dk;
|
||||||
|
return { wrap, dk };
|
||||||
|
});
|
||||||
|
|
||||||
|
if (append) {
|
||||||
|
for (let i = items.length - 1; i >= 0; i--) {
|
||||||
|
const { wrap, dk } = items[i];
|
||||||
|
const target = sentinel.nextSibling;
|
||||||
|
if (dk && dk !== lastKey) {
|
||||||
|
root.insertBefore(makeDateSep(formatDateHeader(payload.messages[i].date)), target);
|
||||||
|
lastKey = dk;
|
||||||
|
}
|
||||||
|
root.insertBefore(wrap, target);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (let i = 0; i < items.length; i++) {
|
||||||
|
const { wrap, dk } = items[i];
|
||||||
|
if (dk && dk !== lastKey) {
|
||||||
|
root.appendChild(makeDateSep(formatDateHeader(payload.messages[i].date)));
|
||||||
|
lastKey = dk;
|
||||||
|
}
|
||||||
|
root.appendChild(wrap);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (payload.messages.length) {
|
||||||
|
viewerState.oldestMessageId = payload.messages[0].message_id;
|
||||||
|
viewerState.newestMessageId = payload.messages[payload.messages.length - 1].message_id;
|
||||||
|
} else if (!append) {
|
||||||
|
viewerState.oldestMessageId = null;
|
||||||
|
viewerState.newestMessageId = null;
|
||||||
|
const empty = document.createElement('div');
|
||||||
|
empty.className = 'viewer-empty-state';
|
||||||
|
empty.textContent = viewerState.search
|
||||||
|
? `No messages found for "${viewerState.search}".`
|
||||||
|
: 'No saved messages in this channel yet.';
|
||||||
|
root.appendChild(empty);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!append) {
|
||||||
|
requestAnimationFrame(() => scrollToBottom());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadChannel(channelId, append = false) {
|
||||||
|
if (viewerState.loading) return;
|
||||||
|
viewerState.loading = true;
|
||||||
|
viewerState.channelId = channelId;
|
||||||
|
renderChannelList();
|
||||||
|
|
||||||
|
const before = append && viewerState.oldestMessageId ? `&before=${viewerState.oldestMessageId}` : '';
|
||||||
|
try {
|
||||||
|
const payload = await api(messageEndpoint(channelId, before));
|
||||||
|
if (append) {
|
||||||
|
const list = document.getElementById('messages-list');
|
||||||
|
const prevScrollHeight = list.scrollHeight;
|
||||||
|
renderMessages(payload, true);
|
||||||
|
requestAnimationFrame(() => {
|
||||||
|
list.scrollTop = list.scrollHeight - prevScrollHeight;
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
renderMessages(payload, false);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to load messages:', err);
|
||||||
|
} finally {
|
||||||
|
viewerState.loading = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshCurrentChannel() {
|
||||||
|
if (!viewerState.channelId || viewerState.loading) return;
|
||||||
|
viewerState.oldestMessageId = null;
|
||||||
|
viewerState.newestMessageId = null;
|
||||||
|
await loadChannel(viewerState.channelId);
|
||||||
|
}
|
||||||
|
|
||||||
|
function setupViewerTools() {
|
||||||
|
const searchInput = document.getElementById('viewer-search');
|
||||||
|
const refreshBtn = document.getElementById('viewer-refresh-btn');
|
||||||
|
const autoRefresh = document.getElementById('viewer-auto-refresh');
|
||||||
|
let searchTimer = null;
|
||||||
|
|
||||||
|
if (searchInput) {
|
||||||
|
searchInput.addEventListener('input', () => {
|
||||||
|
clearTimeout(searchTimer);
|
||||||
|
searchTimer = setTimeout(() => {
|
||||||
|
viewerState.search = searchInput.value.trim();
|
||||||
|
refreshCurrentChannel();
|
||||||
|
}, 250);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (refreshBtn) {
|
||||||
|
refreshBtn.addEventListener('click', () => refreshCurrentChannel());
|
||||||
|
}
|
||||||
|
|
||||||
|
viewerState.autoRefreshTimer = window.setInterval(() => {
|
||||||
|
if (!autoRefresh || autoRefresh.checked) {
|
||||||
|
refreshCurrentChannel();
|
||||||
|
}
|
||||||
|
}, 30000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function setupInfiniteScroll() {
|
||||||
|
const sentinel = document.getElementById('scroll-sentinel');
|
||||||
|
if (!sentinel) return;
|
||||||
|
if (viewerState.sentinelObserver) {
|
||||||
|
viewerState.sentinelObserver.disconnect();
|
||||||
|
}
|
||||||
|
viewerState.sentinelObserver = new IntersectionObserver(
|
||||||
|
(entries) => {
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (entry.isIntersecting && viewerState.channelId && viewerState.oldestMessageId && !viewerState.loading) {
|
||||||
|
loadChannel(viewerState.channelId, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ root: document.getElementById('messages-list'), threshold: 0.1 },
|
||||||
|
);
|
||||||
|
viewerState.sentinelObserver.observe(sentinel);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const params = new URLSearchParams(window.location.search);
|
||||||
|
await loadViewerAccount(params);
|
||||||
|
viewerState.channels = await api(channelsEndpoint());
|
||||||
|
const requestedChannel = params.get('channel');
|
||||||
|
const requestedChannelExists =
|
||||||
|
requestedChannel && viewerState.channels.some((channel) => channel.channel_id === requestedChannel);
|
||||||
|
viewerState.channelId = requestedChannelExists ? requestedChannel : viewerState.channels[0]?.channel_id || null;
|
||||||
|
renderChannelList();
|
||||||
|
|
||||||
|
if (viewerState.channelId) {
|
||||||
|
await loadChannel(viewerState.channelId);
|
||||||
|
} else {
|
||||||
|
resetMessageView(
|
||||||
|
'No channels',
|
||||||
|
viewerState.accountId ? 'This account has no tracked channels yet.' : 'No local legacy channels found.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
setupInfiniteScroll();
|
||||||
|
setupViewerTools();
|
||||||
|
|
||||||
|
const toggle = document.getElementById('sidebar-toggle');
|
||||||
|
const sidebar = document.querySelector('.viewer-sidebar');
|
||||||
|
if (toggle && sidebar) {
|
||||||
|
toggle.addEventListener('click', () => {
|
||||||
|
sidebar.classList.toggle('open');
|
||||||
|
});
|
||||||
|
sidebar.addEventListener('click', (e) => {
|
||||||
|
if (e.target === sidebar) {
|
||||||
|
sidebar.classList.remove('open');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error(error);
|
||||||
|
showToast(error.message, 'error');
|
||||||
|
});
|
||||||
+2840
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user