forust/telegram-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Activity

Merge dev into main: lint fixes and config updates

Browse Source
This commit is contained in:
mr-forust
2026-06-19 11:52:34 +02:00
parent c5361f2271 ac5beb831b
commit f5ef74bda6
19 changed files with 1567 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.editorconfig
+24
View File
@@ -0,0 +1,24 @@
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.{yml,yaml}]
indent_size = 2
[*.{json,jsonc}]
indent_size = 2
[*.md]
trim_trailing_whitespace = false
[*.py]
indent_size = 4
[{Makefile,makefile}]
indent_style = tab
.hadolint.yaml
+10
View File
@@ -0,0 +1,10 @@
ignored:
- DL3008
- DL3042
- DL3018
- DL3059
trustedRegistries:
- docker.io
- ghcr.io
- quay.io
- gcr.forust.xyz
.markdownlint.json
+8
View File
@@ -0,0 +1,8 @@
{
"default": true,
"MD013": false,
"MD024": false,
"MD033": false,
"MD041": false,
"MD046": false
}
.prettierrc.yaml
+8
View File
@@ -0,0 +1,8 @@
bracketSameLine: true
htmlWhitespaceSensitivity: css
printWidth: 120
tabWidth: 2
singleQuote: true
trailingComma: all
proseWrap: preserve
endOfLine: lf
.vscode/settings.json
Vendored
+34
View File
@@ -0,0 +1,34 @@
{
"[yaml]": {
"editor.tabSize": 2,
"editor.insertSpaces": true,
"editor.formatOnSave": true,
"editor.defaultFormatter": "redhat.vscode-yaml"
},
"[python]": {
"editor.tabSize": 4,
"editor.formatOnSave": true,
"editor.defaultFormatter": "charliermarsh.ruff"
},
"[dockerfile]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "exiasr.hadolint"
},
"[markdown]": {
"editor.wordWrap": "wordWrapColumn",
"editor.wordWrapColumn": 120
},
"yaml.schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"yaml.format.enable": true,
"yaml.validate": true,
"yaml.completion": true,
"prettier.bracketSameLine": true,
"prettier.htmlWhitespaceSensitivity": "css",
"prettier.printWidth": 120,
"editor.formatOnSave": true,
"files.autoSave": "onFocusChange",
"editor.wordWrapColumn": 120,
"editor.wordWrap": "wordWrapColumn"
}
.yamllint
+17
View File
@@ -0,0 +1,17 @@
extends: default
rules:
comments:
min-spaces-from-content: 1
comments-indentation: false
document-start: disable
line-length: disable
braces:
min-spaces-inside: 0
max-spaces-inside: 1
brackets:
min-spaces-inside: 0
max-spaces-inside: 1
indentation:
spaces: 2
indent-sequences: consistent
.zed/settings.json
+39
View File
@@ -0,0 +1,39 @@
{
"tab_size": 2,
"soft_wrap": "prefer_line",
"preferred_line_length": 120,
"format_on_save": "on",
"languages": {
"YAML": {
"tab_size": 2,
"hard_tabs": false,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "yaml-language-server" }
}
},
"Python": {
"tab_size": 4,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "ruff" }
}
}
},
"lsp": {
"yaml-language-server": {
"settings": {
"yaml": {
"schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"validate": true,
"completion": true,
"format": {
"enable": true
}
}
}
}
}
}
app_state.py
+6 -2
View File
@@ -57,7 +57,9 @@ class StateStore:
self.path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8") as handle:
json.dump(self._merge_defaults(state), handle, ensure_ascii=False, indent=2)
json.dump(
self._merge_defaults(state), handle, ensure_ascii=False, indent=2
)
handle.write("\n")
tmp_path.replace(self.path)
self._cache = None
@@ -71,7 +73,9 @@ class StateStore:
def continuous_config(self) -> Dict[str, Any]:
state = self.load()
return dict(state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"])
return dict(
state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"]
)
def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
def mutate(state: Dict[str, Any]) -> None:
compose.yaml
+7
View File
@@ -4,11 +4,18 @@ services:
context: .
dockerfile: Dockerfile
container_name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
user: 1000:1000
restart: unless-stopped
tty: true
ports:
- "7887:8080"
healthcheck:
test: ["CMD", "python", "-c", "import http.client,json;c=http.client.HTTPConnection('localhost',8080);c.request('GET','/health');r=c.getresponse();exit(0)if json.loads(r.read()).get('ok')else exit(1)"]
interval: 30s
timeout: 10s
retries: 3
start_period: 15s
volumes:
- ./data:/app/data
- ./session:/app/session
k8s/telegram-scraper.yaml
+87
View File
@@ -0,0 +1,87 @@
apiVersion: v1
kind: Namespace
metadata:
name: telegram-scraper
---
apiVersion: v1
kind: Service
metadata:
name: telegram-scraper-service
namespace: telegram-scraper
spec:
selector:
app: telegram-scraper
ports:
- port: 8080
targetPort: 8080
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: telegram-scraper-statefulset
namespace: telegram-scraper
spec:
selector:
matchLabels:
app: telegram-scraper
serviceName: telegram-scraper-service
replicas: 1
template:
metadata:
labels:
app: telegram-scraper
spec:
containers:
- name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
tty: true
stdin: true
ports:
- containerPort: 8080
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 15
periodSeconds: 30
readinessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: telegram-scraper-data
mountPath: /app/data
- name: telegram-scraper-session
mountPath: /app/session
volumeClaimTemplates:
- metadata:
name: telegram-scraper-data
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 20Gi
- metadata:
name: telegram-scraper-session
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 5Mi
---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: telegram-scraper-route
namespace: telegram-scraper
spec:
entryPoints:
- websecure
routes:
- match: Host(`tg.workstation.internal`)
kind: Rule
services:
- name: telegram-scraper-service
port: 8080
scraper_jobs.py
+2 -1
View File
@@ -41,7 +41,8 @@ class ScraperJobService:
)
elif job_type == "scrape_selected":
await self._scrape_channels(
scraper, [str(channel_id) for channel_id in payload.get("channels", [])]
scraper,
[str(channel_id) for channel_id in payload.get("channels", [])],
)
elif job_type == "export_all":
await scraper.export_data()
scripts/smoke_test.py
+5 -1
View File
@@ -60,7 +60,11 @@ def main() -> int:
status, body = fetch(endpoint)
if status != 200:
raise RuntimeError(f"{endpoint} returned HTTP {status}")
if endpoint.endswith(".json") or endpoint.startswith("/api") or endpoint.startswith("/health"):
if (
endpoint.endswith(".json")
or endpoint.startswith("/api")
or endpoint.startswith("/health")
):
json.loads(body.decode("utf-8"))
print(f"ok {endpoint}")
return 0
telegram-scraper-OLD.py
+364 -188
View File
File diff suppressed because it is too large Load Diff
telegram_scraper_with_forwarding.py
+591 -325
View File
File diff suppressed because it is too large Load Diff
webui/index.html
+1 -1
View File
@@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Control Panel</title>
<link rel="stylesheet" href="/static/style.css" />
<link rel="stylesheet" href="/static/style.css?v=2" />
</head>
<body>
<div class="app-shell">
webui/style.css
+115 -5
View File
@@ -47,7 +47,8 @@ button {
}
.app-shell {
min-height: 100vh;
height: 100vh;
overflow: hidden;
display: grid;
grid-template-columns: 300px minmax(0, 1fr);
}
@@ -59,6 +60,7 @@ button {
}
.content {
overflow-y: auto;
padding: 28px;
max-width: 1320px;
width: 100%;
@@ -494,6 +496,10 @@ tbody tr:hover {
overflow-y: auto;
}
.sidebar-toggle {
display: none;
}
.viewer-sidebar-head {
display: flex;
align-items: flex-start;
@@ -814,6 +820,10 @@ tbody tr:hover {
grid-template-columns: 1fr;
}
.app-shell {
grid-template-rows: auto 1fr;
}
.sidebar,
.viewer-sidebar {
border-right: 0;
@@ -825,14 +835,114 @@ tbody tr:hover {
}
}
@media (max-width: 720px) {
.content,
.viewer-main,
.sidebar,
/* ---------- Mobile: tablet & phone ---------- */
@media (max-width: 768px) {
.sidebar-toggle {
display: inline-flex;
}
.viewer-sidebar {
position: fixed;
left: -300px;
top: 0;
bottom: 0;
width: 280px;
z-index: 100;
transition: left 0.2s ease;
border-right: 1px dashed var(--line);
border-bottom: 0;
}
.viewer-sidebar.open {
left: 0;
}
.viewer-sidebar.open::before {
content: "";
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.5);
z-index: -1;
}
.viewer-header {
padding: 12px 14px;
}
.messages-list {
padding: 8px 12px 16px;
}
.chat-bubble {
max-width: 92%;
}
.chat-bubble::before {
display: none;
}
.content {
padding: 18px;
}
.viewer-sidebar-head h1 {
font-size: 1.2rem;
}
}
@media (max-width: 480px) {
.content {
padding: 12px;
}
.viewer-main {
padding: 0;
}
.viewer-header {
padding: 10px 10px;
}
.viewer-header h2 {
font-size: 1.1rem;
}
.messages-list {
padding: 6px 8px 12px;
}
.chat-bubble {
max-width: 96%;
padding: 6px 9px;
font-size: 0.92rem;
}
.chat-footer {
font-size: 0.65rem;
}
.chat-sender {
font-size: 0.75rem;
}
.chat-reply-author {
font-size: 0.72rem;
}
.chat-reply-text {
font-size: 0.75rem;
}
.sidebar {
padding: 16px;
}
.viewer-sidebar {
width: 260px;
padding: 16px;
}
.panel-header,
.viewer-header,
.dialog-header {
webui/viewer.html
+2 -1
View File
@@ -4,7 +4,7 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Viewer</title>
<link rel="stylesheet" href="/static/style.css" />
<link rel="stylesheet" href="/static/style.css?v=2" />
</head>
<body>
<div class="viewer-shell">
@@ -14,6 +14,7 @@
<div class="eyebrow">Export Viewer</div>
<h1>Messages</h1>
</div>
<button id="sidebar-toggle" class="button button-small sidebar-toggle">☰</button>
<a class="button button-small" href="/">Dashboard</a>
</div>
<div id="viewer-channel-list" class="viewer-channel-list"></div>
webui/viewer.js
+14
View File
@@ -81,6 +81,7 @@ function renderChannelList() {
node.classList.add("active");
}
node.addEventListener("click", () => {
document.querySelector(".viewer-sidebar")?.classList.remove("open");
viewerState.oldestMessageId = null;
viewerState.newestMessageId = null;
loadChannel(channel.channel_id);
@@ -304,6 +305,19 @@ async function main() {
}
setupInfiniteScroll();
const toggle = document.getElementById("sidebar-toggle");
const sidebar = document.querySelector(".viewer-sidebar");
if (toggle && sidebar) {
toggle.addEventListener("click", () => {
sidebar.classList.toggle("open");
});
sidebar.addEventListener("click", (e) => {
if (e.target === sidebar) {
sidebar.classList.remove("open");
}
});
}
}
main().catch((error) => {
webui_server.py
+35 -13
View File
@@ -306,7 +306,9 @@ class JobRunner:
if job.status == "running":
running_jobs.append(job)
if running_jobs:
logger.warning("Waiting for %d running job(s) to finish...", len(running_jobs))
logger.warning(
"Waiting for %d running job(s) to finish...", len(running_jobs)
)
deadline = time.time() + timeout
while time.time() < deadline:
with self.lock:
@@ -561,6 +563,7 @@ class TelegramAuthManager:
async def _disconnect():
if self.client:
await self.client.disconnect()
if self.client:
try:
future = asyncio.run_coroutine_threadsafe(_disconnect(), self.loop)
@@ -831,7 +834,10 @@ def openapi_payload() -> Dict[str, Any]:
"requestBody": json_body(
{
"api_id": {"type": "integer", "example": 123456},
"api_hash": {"type": "string", "example": "0123456789abcdef"},
"api_hash": {
"type": "string",
"example": "0123456789abcdef",
},
},
["api_id", "api_hash"],
),
@@ -929,7 +935,11 @@ def openapi_payload() -> Dict[str, Any]:
{
"name": "limit",
"in": "query",
"schema": {"type": "integer", "default": 120, "maximum": 300},
"schema": {
"type": "integer",
"default": 120,
"maximum": 300,
},
},
{
"name": "before",
@@ -945,7 +955,10 @@ def openapi_payload() -> Dict[str, Any]:
"summary": "Track a channel",
"requestBody": json_body(
{
"channel_id": {"type": "string", "example": "-1001234567890"},
"channel_id": {
"type": "string",
"example": "-1001234567890",
},
"name": {"type": "string", "example": "Research feed"},
},
["channel_id"],
@@ -990,7 +1003,10 @@ def openapi_payload() -> Dict[str, Any]:
"schema": {"type": "string"},
}
],
"responses": {**json_response, "404": {"description": "Job not found"}},
"responses": {
**json_response,
"404": {"description": "Job not found"},
},
}
},
"/api/jobs/scrape": {
@@ -1416,15 +1432,18 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
try:
start, end = self._parse_range(range_header, file_size)
except ValueError:
self.send_error_json(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range")
self.send_error_json(
HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range"
)
return
content_length = end - start + 1
self.send_response(HTTPStatus.PARTIAL_CONTENT)
self.send_header("Content-Type", content_type)
self.send_header("Accept-Ranges", "bytes")
self.send_header("Last-Modified", time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
))
self.send_header(
"Last-Modified",
time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
)
self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
self.send_header("Content-Length", str(content_length))
self.end_headers()
@@ -1434,9 +1453,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", content_type)
self.send_header("Accept-Ranges", "bytes")
self.send_header("Last-Modified", time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
))
self.send_header(
"Last-Modified",
time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
)
self.send_header("Content-Length", str(file_size))
self.end_headers()
if not head_only:
@@ -1495,7 +1515,9 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
self.job_runner = JobRunner()
self.auth_manager = TelegramAuthManager()
self.continuous_manager = ContinuousScrapeManager()
if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get("enabled", True):
if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get(
"enabled", True
):
self.continuous_manager.start()