forust/telegram-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Activity

lint: fix bare except and unused variables

Browse Source
This commit is contained in:
mr-forust
2026-06-19 11:52:30 +02:00
parent df644219de
commit ac5beb831b
15 changed files with 1359 additions and 728 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.editorconfig
+24
View File
@@ -0,0 +1,24 @@
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.{yml,yaml}]
indent_size = 2
[*.{json,jsonc}]
indent_size = 2
[*.md]
trim_trailing_whitespace = false
[*.py]
indent_size = 4
[{Makefile,makefile}]
indent_style = tab
.hadolint.yaml
+10
View File
@@ -0,0 +1,10 @@
ignored:
- DL3008
- DL3042
- DL3018
- DL3059
trustedRegistries:
- docker.io
- ghcr.io
- quay.io
- gcr.forust.xyz
.markdownlint.json
+8
View File
@@ -0,0 +1,8 @@
{
"default": true,
"MD013": false,
"MD024": false,
"MD033": false,
"MD041": false,
"MD046": false
}
.prettierrc.yaml
+8
View File
@@ -0,0 +1,8 @@
bracketSameLine: true
htmlWhitespaceSensitivity: css
printWidth: 120
tabWidth: 2
singleQuote: true
trailingComma: all
proseWrap: preserve
endOfLine: lf
.vscode/settings.json
Vendored
+34
View File
@@ -0,0 +1,34 @@
{
"[yaml]": {
"editor.tabSize": 2,
"editor.insertSpaces": true,
"editor.formatOnSave": true,
"editor.defaultFormatter": "redhat.vscode-yaml"
},
"[python]": {
"editor.tabSize": 4,
"editor.formatOnSave": true,
"editor.defaultFormatter": "charliermarsh.ruff"
},
"[dockerfile]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "exiasr.hadolint"
},
"[markdown]": {
"editor.wordWrap": "wordWrapColumn",
"editor.wordWrapColumn": 120
},
"yaml.schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"yaml.format.enable": true,
"yaml.validate": true,
"yaml.completion": true,
"prettier.bracketSameLine": true,
"prettier.htmlWhitespaceSensitivity": "css",
"prettier.printWidth": 120,
"editor.formatOnSave": true,
"files.autoSave": "onFocusChange",
"editor.wordWrapColumn": 120,
"editor.wordWrap": "wordWrapColumn"
}
.yamllint
+17
View File
@@ -0,0 +1,17 @@
extends: default
rules:
comments:
min-spaces-from-content: 1
comments-indentation: false
document-start: disable
line-length: disable
braces:
min-spaces-inside: 0
max-spaces-inside: 1
brackets:
min-spaces-inside: 0
max-spaces-inside: 1
indentation:
spaces: 2
indent-sequences: consistent
.zed/settings.json
+39
View File
@@ -0,0 +1,39 @@
{
"tab_size": 2,
"soft_wrap": "prefer_line",
"preferred_line_length": 120,
"format_on_save": "on",
"languages": {
"YAML": {
"tab_size": 2,
"hard_tabs": false,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "yaml-language-server" }
}
},
"Python": {
"tab_size": 4,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "ruff" }
}
}
},
"lsp": {
"yaml-language-server": {
"settings": {
"yaml": {
"schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"validate": true,
"completion": true,
"format": {
"enable": true
}
}
}
}
}
}
app_state.py
+6 -2
View File
@@ -57,7 +57,9 @@ class StateStore:
self.path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8") as handle:
json.dump(self._merge_defaults(state), handle, ensure_ascii=False, indent=2)
json.dump(
self._merge_defaults(state), handle, ensure_ascii=False, indent=2
)
handle.write("\n")
tmp_path.replace(self.path)
self._cache = None
@@ -71,7 +73,9 @@ class StateStore:
def continuous_config(self) -> Dict[str, Any]:
state = self.load()
return dict(state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"])
return dict(
state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"]
)
def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
def mutate(state: Dict[str, Any]) -> None:
compose.yaml
+6
View File
@@ -10,6 +10,12 @@ services:
tty: true
ports:
- "7887:8080"
healthcheck:
test: ["CMD", "python", "-c", "import http.client,json;c=http.client.HTTPConnection('localhost',8080);c.request('GET','/health');r=c.getresponse();exit(0)if json.loads(r.read()).get('ok')else exit(1)"]
interval: 30s
timeout: 10s
retries: 3
start_period: 15s
volumes:
- ./data:/app/data
- ./session:/app/session
k8s/telegram-scraper.yaml
+12
View File
@@ -38,6 +38,18 @@ spec:
stdin: true
ports:
- containerPort: 8080
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 15
periodSeconds: 30
readinessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: telegram-scraper-data
mountPath: /app/data
scraper_jobs.py
+2 -1
View File
@@ -41,7 +41,8 @@ class ScraperJobService:
)
elif job_type == "scrape_selected":
await self._scrape_channels(
scraper, [str(channel_id) for channel_id in payload.get("channels", [])]
scraper,
[str(channel_id) for channel_id in payload.get("channels", [])],
)
elif job_type == "export_all":
await scraper.export_data()
scripts/smoke_test.py
+5 -1
View File
@@ -60,7 +60,11 @@ def main() -> int:
status, body = fetch(endpoint)
if status != 200:
raise RuntimeError(f"{endpoint} returned HTTP {status}")
if endpoint.endswith(".json") or endpoint.startswith("/api") or endpoint.startswith("/health"):
if (
endpoint.endswith(".json")
or endpoint.startswith("/api")
or endpoint.startswith("/health")
):
json.loads(body.decode("utf-8"))
print(f"ok {endpoint}")
return 0
telegram-scraper-OLD.py
+364 -188
View File
File diff suppressed because it is too large Load Diff
telegram_scraper_with_forwarding.py
+591 -325
View File
File diff suppressed because it is too large Load Diff
webui_server.py
+35 -13
View File
@@ -306,7 +306,9 @@ class JobRunner:
if job.status == "running":
running_jobs.append(job)
if running_jobs:
logger.warning("Waiting for %d running job(s) to finish...", len(running_jobs))
logger.warning(
"Waiting for %d running job(s) to finish...", len(running_jobs)
)
deadline = time.time() + timeout
while time.time() < deadline:
with self.lock:
@@ -561,6 +563,7 @@ class TelegramAuthManager:
async def _disconnect():
if self.client:
await self.client.disconnect()
if self.client:
try:
future = asyncio.run_coroutine_threadsafe(_disconnect(), self.loop)
@@ -831,7 +834,10 @@ def openapi_payload() -> Dict[str, Any]:
"requestBody": json_body(
{
"api_id": {"type": "integer", "example": 123456},
"api_hash": {"type": "string", "example": "0123456789abcdef"},
"api_hash": {
"type": "string",
"example": "0123456789abcdef",
},
},
["api_id", "api_hash"],
),
@@ -929,7 +935,11 @@ def openapi_payload() -> Dict[str, Any]:
{
"name": "limit",
"in": "query",
"schema": {"type": "integer", "default": 120, "maximum": 300},
"schema": {
"type": "integer",
"default": 120,
"maximum": 300,
},
},
{
"name": "before",
@@ -945,7 +955,10 @@ def openapi_payload() -> Dict[str, Any]:
"summary": "Track a channel",
"requestBody": json_body(
{
"channel_id": {"type": "string", "example": "-1001234567890"},
"channel_id": {
"type": "string",
"example": "-1001234567890",
},
"name": {"type": "string", "example": "Research feed"},
},
["channel_id"],
@@ -990,7 +1003,10 @@ def openapi_payload() -> Dict[str, Any]:
"schema": {"type": "string"},
}
],
"responses": {**json_response, "404": {"description": "Job not found"}},
"responses": {
**json_response,
"404": {"description": "Job not found"},
},
}
},
"/api/jobs/scrape": {
@@ -1416,15 +1432,18 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
try:
start, end = self._parse_range(range_header, file_size)
except ValueError:
self.send_error_json(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range")
self.send_error_json(
HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range"
)
return
content_length = end - start + 1
self.send_response(HTTPStatus.PARTIAL_CONTENT)
self.send_header("Content-Type", content_type)
self.send_header("Accept-Ranges", "bytes")
self.send_header("Last-Modified", time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
))
self.send_header(
"Last-Modified",
time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
)
self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
self.send_header("Content-Length", str(content_length))
self.end_headers()
@@ -1434,9 +1453,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", content_type)
self.send_header("Accept-Ranges", "bytes")
self.send_header("Last-Modified", time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
))
self.send_header(
"Last-Modified",
time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
)
self.send_header("Content-Length", str(file_size))
self.end_headers()
if not head_only:
@@ -1495,7 +1515,9 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
self.job_runner = JobRunner()
self.auth_manager = TelegramAuthManager()
self.continuous_manager = ContinuousScrapeManager()
if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get("enabled", True):
if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get(
"enabled", True
):
self.continuous_manager.start()