lint: fix bare except and unused variables

.editorconfig

@@ -0,0 +1,24 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.{json,jsonc}]
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.py]
+indent_size = 4
+
+[{Makefile,makefile}]
+indent_style = tab

.hadolint.yaml

@@ -0,0 +1,10 @@
+ignored:
+  - DL3008
+  - DL3042
+  - DL3018
+  - DL3059
+trustedRegistries:
+  - docker.io
+  - ghcr.io
+  - quay.io
+  - gcr.forust.xyz

.markdownlint.json

@@ -0,0 +1,8 @@
+{
+  "default": true,
+  "MD013": false,
+  "MD024": false,
+  "MD033": false,
+  "MD041": false,
+  "MD046": false
+}

.prettierrc.yaml

@@ -0,0 +1,8 @@
+bracketSameLine: true
+htmlWhitespaceSensitivity: css
+printWidth: 120
+tabWidth: 2
+singleQuote: true
+trailingComma: all
+proseWrap: preserve
+endOfLine: lf

.vscode/settings.json

@@ -0,0 +1,34 @@
+{
+  "[yaml]": {
+    "editor.tabSize": 2,
+    "editor.insertSpaces": true,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "redhat.vscode-yaml"
+  },
+  "[python]": {
+    "editor.tabSize": 4,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
+  "[dockerfile]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "exiasr.hadolint"
+  },
+  "[markdown]": {
+    "editor.wordWrap": "wordWrapColumn",
+    "editor.wordWrapColumn": 120
+  },
+  "yaml.schemas": {
+    "kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
+  },
+  "yaml.format.enable": true,
+  "yaml.validate": true,
+  "yaml.completion": true,
+  "prettier.bracketSameLine": true,
+  "prettier.htmlWhitespaceSensitivity": "css",
+  "prettier.printWidth": 120,
+  "editor.formatOnSave": true,
+  "files.autoSave": "onFocusChange",
+  "editor.wordWrapColumn": 120,
+  "editor.wordWrap": "wordWrapColumn"
+}

.yamllint

@@ -0,0 +1,17 @@
+extends: default
+
+rules:
+  comments:
+    min-spaces-from-content: 1
+  comments-indentation: false
+  document-start: disable
+  line-length: disable
+  braces:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  brackets:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  indentation:
+    spaces: 2
+    indent-sequences: consistent

.zed/settings.json

@@ -0,0 +1,39 @@
+{
+  "tab_size": 2,
+  "soft_wrap": "prefer_line",
+  "preferred_line_length": 120,
+  "format_on_save": "on",
+  "languages": {
+    "YAML": {
+      "tab_size": 2,
+      "hard_tabs": false,
+      "format_on_save": "on",
+      "formatter": {
+        "language_server": { "name": "yaml-language-server" }
+      }
+    },
+    "Python": {
+      "tab_size": 4,
+      "format_on_save": "on",
+      "formatter": {
+        "language_server": { "name": "ruff" }
+      }
+    }
+  },
+  "lsp": {
+    "yaml-language-server": {
+      "settings": {
+        "yaml": {
+          "schemas": {
+            "kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
+          },
+          "validate": true,
+          "completion": true,
+          "format": {
+            "enable": true
+          }
+        }
+      }
+    }
+  }
+}

app_state.py

@@ -57,7 +57,9 @@ class StateStore:
             self.path.parent.mkdir(parents=True, exist_ok=True)
             tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
             with tmp_path.open("w", encoding="utf-8") as handle:
-                json.dump(self._merge_defaults(state), handle, ensure_ascii=False, indent=2)
+                json.dump(
+                    self._merge_defaults(state), handle, ensure_ascii=False, indent=2
+                )
                 handle.write("\n")
             tmp_path.replace(self.path)
             self._cache = None
@@ -71,7 +73,9 @@ class StateStore:
 
     def continuous_config(self) -> Dict[str, Any]:
         state = self.load()
-        return dict(state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"])
+        return dict(
+            state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"]
+        )
 
     def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
         def mutate(state: Dict[str, Any]) -> None:

compose.yaml

@@ -10,6 +10,12 @@ services:
     tty: true
     ports:
       - "7887:8080"
+    healthcheck:
+      test: ["CMD", "python", "-c", "import http.client,json;c=http.client.HTTPConnection('localhost',8080);c.request('GET','/health');r=c.getresponse();exit(0)if json.loads(r.read()).get('ok')else exit(1)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 15s
     volumes:
       - ./data:/app/data
       - ./session:/app/session

k8s/telegram-scraper.yaml

@@ -38,6 +38,18 @@ spec:
           stdin: true
           ports:
             - containerPort: 8080
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 10
           volumeMounts:
             - name: telegram-scraper-data
               mountPath: /app/data

scraper_jobs.py

@@ -41,7 +41,8 @@ class ScraperJobService:
                 )
             elif job_type == "scrape_selected":
                 await self._scrape_channels(
-                    scraper, [str(channel_id) for channel_id in payload.get("channels", [])]
+                    scraper,
+                    [str(channel_id) for channel_id in payload.get("channels", [])],
                 )
             elif job_type == "export_all":
                 await scraper.export_data()

scripts/smoke_test.py

@@ -60,7 +60,11 @@ def main() -> int:
             status, body = fetch(endpoint)
             if status != 200:
                 raise RuntimeError(f"{endpoint} returned HTTP {status}")
-            if endpoint.endswith(".json") or endpoint.startswith("/api") or endpoint.startswith("/health"):
+            if (
+                endpoint.endswith(".json")
+                or endpoint.startswith("/api")
+                or endpoint.startswith("/health")
+            ):
                 json.loads(body.decode("utf-8"))
             print(f"ok {endpoint}")
         return 0

webui_server.py

@@ -306,7 +306,9 @@ class JobRunner:
                 if job.status == "running":
                     running_jobs.append(job)
         if running_jobs:
-            logger.warning("Waiting for %d running job(s) to finish...", len(running_jobs))
+            logger.warning(
+                "Waiting for %d running job(s) to finish...", len(running_jobs)
+            )
             deadline = time.time() + timeout
             while time.time() < deadline:
                 with self.lock:
@@ -561,6 +563,7 @@ class TelegramAuthManager:
         async def _disconnect():
             if self.client:
                 await self.client.disconnect()
+
         if self.client:
             try:
                 future = asyncio.run_coroutine_threadsafe(_disconnect(), self.loop)
@@ -831,7 +834,10 @@ def openapi_payload() -> Dict[str, Any]:
                     "requestBody": json_body(
                         {
                             "api_id": {"type": "integer", "example": 123456},
-                            "api_hash": {"type": "string", "example": "0123456789abcdef"},
+                            "api_hash": {
+                                "type": "string",
+                                "example": "0123456789abcdef",
+                            },
                         },
                         ["api_id", "api_hash"],
                     ),
@@ -929,7 +935,11 @@ def openapi_payload() -> Dict[str, Any]:
                         {
                             "name": "limit",
                             "in": "query",
-                            "schema": {"type": "integer", "default": 120, "maximum": 300},
+                            "schema": {
+                                "type": "integer",
+                                "default": 120,
+                                "maximum": 300,
+                            },
                         },
                         {
                             "name": "before",
@@ -945,7 +955,10 @@ def openapi_payload() -> Dict[str, Any]:
                     "summary": "Track a channel",
                     "requestBody": json_body(
                         {
-                            "channel_id": {"type": "string", "example": "-1001234567890"},
+                            "channel_id": {
+                                "type": "string",
+                                "example": "-1001234567890",
+                            },
                             "name": {"type": "string", "example": "Research feed"},
                         },
                         ["channel_id"],
@@ -990,7 +1003,10 @@ def openapi_payload() -> Dict[str, Any]:
                             "schema": {"type": "string"},
                         }
                     ],
-                    "responses": {**json_response, "404": {"description": "Job not found"}},
+                    "responses": {
+                        **json_response,
+                        "404": {"description": "Job not found"},
+                    },
                 }
             },
             "/api/jobs/scrape": {
@@ -1416,15 +1432,18 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             try:
                 start, end = self._parse_range(range_header, file_size)
             except ValueError:
-                self.send_error_json(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range")
+                self.send_error_json(
+                    HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range"
+                )
                 return
             content_length = end - start + 1
             self.send_response(HTTPStatus.PARTIAL_CONTENT)
             self.send_header("Content-Type", content_type)
             self.send_header("Accept-Ranges", "bytes")
-            self.send_header("Last-Modified", time.strftime(
+            self.send_header(
-                "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
+                "Last-Modified",
-            ))
+                time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
+            )
             self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
             self.send_header("Content-Length", str(content_length))
             self.end_headers()
@@ -1434,9 +1453,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             self.send_response(HTTPStatus.OK)
             self.send_header("Content-Type", content_type)
             self.send_header("Accept-Ranges", "bytes")
-            self.send_header("Last-Modified", time.strftime(
+            self.send_header(
-                "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
+                "Last-Modified",
-            ))
+                time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
+            )
             self.send_header("Content-Length", str(file_size))
             self.end_headers()
             if not head_only:
@@ -1495,7 +1515,9 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
         self.job_runner = JobRunner()
         self.auth_manager = TelegramAuthManager()
         self.continuous_manager = ContinuousScrapeManager()
-        if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get("enabled", True):
+        if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get(
+            "enabled", True
+        ):
             self.continuous_manager.start()