Merge dev into main: lint fixes and config updates

.editorconfig

@@ -0,0 +1,24 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.{json,jsonc}]
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.py]
+indent_size = 4
+
+[{Makefile,makefile}]
+indent_style = tab

.hadolint.yaml

@@ -0,0 +1,10 @@
+ignored:
+  - DL3008
+  - DL3042
+  - DL3018
+  - DL3059
+trustedRegistries:
+  - docker.io
+  - ghcr.io
+  - quay.io
+  - gcr.forust.xyz

.markdownlint.json

@@ -0,0 +1,8 @@
+{
+  "default": true,
+  "MD013": false,
+  "MD024": false,
+  "MD033": false,
+  "MD041": false,
+  "MD046": false
+}

.prettierrc.yaml

@@ -0,0 +1,8 @@
+bracketSameLine: true
+htmlWhitespaceSensitivity: css
+printWidth: 120
+tabWidth: 2
+singleQuote: true
+trailingComma: all
+proseWrap: preserve
+endOfLine: lf

.vscode/settings.json

@@ -0,0 +1,34 @@
+{
+  "[yaml]": {
+    "editor.tabSize": 2,
+    "editor.insertSpaces": true,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "redhat.vscode-yaml"
+  },
+  "[python]": {
+    "editor.tabSize": 4,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
+  "[dockerfile]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "exiasr.hadolint"
+  },
+  "[markdown]": {
+    "editor.wordWrap": "wordWrapColumn",
+    "editor.wordWrapColumn": 120
+  },
+  "yaml.schemas": {
+    "kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
+  },
+  "yaml.format.enable": true,
+  "yaml.validate": true,
+  "yaml.completion": true,
+  "prettier.bracketSameLine": true,
+  "prettier.htmlWhitespaceSensitivity": "css",
+  "prettier.printWidth": 120,
+  "editor.formatOnSave": true,
+  "files.autoSave": "onFocusChange",
+  "editor.wordWrapColumn": 120,
+  "editor.wordWrap": "wordWrapColumn"
+}

.yamllint

@@ -0,0 +1,17 @@
+extends: default
+
+rules:
+  comments:
+    min-spaces-from-content: 1
+  comments-indentation: false
+  document-start: disable
+  line-length: disable
+  braces:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  brackets:
+    min-spaces-inside: 0
+    max-spaces-inside: 1
+  indentation:
+    spaces: 2
+    indent-sequences: consistent

.zed/settings.json

@@ -0,0 +1,39 @@
+{
+  "tab_size": 2,
+  "soft_wrap": "prefer_line",
+  "preferred_line_length": 120,
+  "format_on_save": "on",
+  "languages": {
+    "YAML": {
+      "tab_size": 2,
+      "hard_tabs": false,
+      "format_on_save": "on",
+      "formatter": {
+        "language_server": { "name": "yaml-language-server" }
+      }
+    },
+    "Python": {
+      "tab_size": 4,
+      "format_on_save": "on",
+      "formatter": {
+        "language_server": { "name": "ruff" }
+      }
+    }
+  },
+  "lsp": {
+    "yaml-language-server": {
+      "settings": {
+        "yaml": {
+          "schemas": {
+            "kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
+          },
+          "validate": true,
+          "completion": true,
+          "format": {
+            "enable": true
+          }
+        }
+      }
+    }
+  }
+}

app_state.py

@@ -57,7 +57,9 @@ class StateStore:
             self.path.parent.mkdir(parents=True, exist_ok=True)
             tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
             with tmp_path.open("w", encoding="utf-8") as handle:
-                json.dump(self._merge_defaults(state), handle, ensure_ascii=False, indent=2)
+                json.dump(
+                    self._merge_defaults(state), handle, ensure_ascii=False, indent=2
+                )
                 handle.write("\n")
             tmp_path.replace(self.path)
             self._cache = None
@@ -71,7 +73,9 @@ class StateStore:
 
     def continuous_config(self) -> Dict[str, Any]:
         state = self.load()
-        return dict(state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"])
+        return dict(
+            state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"]
+        )
 
     def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
         def mutate(state: Dict[str, Any]) -> None:

compose.yaml

@@ -4,11 +4,18 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: telegram-scraper
+    image: gcr.forust.xyz/forust/telegram-scraper:latest
     user: 1000:1000
     restart: unless-stopped
     tty: true
     ports:
       - "7887:8080"
+    healthcheck:
+      test: ["CMD", "python", "-c", "import http.client,json;c=http.client.HTTPConnection('localhost',8080);c.request('GET','/health');r=c.getresponse();exit(0)if json.loads(r.read()).get('ok')else exit(1)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 15s
     volumes:
       - ./data:/app/data
       - ./session:/app/session

k8s/telegram-scraper.yaml

@@ -0,0 +1,87 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telegram-scraper
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: telegram-scraper-service
+  namespace: telegram-scraper
+spec:
+  selector:
+    app: telegram-scraper
+  ports:
+    - port: 8080
+      targetPort: 8080
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: telegram-scraper-statefulset
+  namespace: telegram-scraper
+spec:
+  selector:
+    matchLabels:
+      app: telegram-scraper
+  serviceName: telegram-scraper-service
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: telegram-scraper
+    spec:
+      containers:
+        - name: telegram-scraper
+          image: gcr.forust.xyz/forust/telegram-scraper:latest
+          tty: true
+          stdin: true
+          ports:
+            - containerPort: 8080
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          volumeMounts:
+            - name: telegram-scraper-data
+              mountPath: /app/data
+            - name: telegram-scraper-session
+              mountPath: /app/session
+  volumeClaimTemplates:
+    - metadata:
+        name: telegram-scraper-data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 20Gi
+    - metadata:
+        name: telegram-scraper-session
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 5Mi
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: telegram-scraper-route
+  namespace: telegram-scraper
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`tg.workstation.internal`)
+      kind: Rule
+      services:
+        - name: telegram-scraper-service
+          port: 8080

scraper_jobs.py

@@ -41,7 +41,8 @@ class ScraperJobService:
                 )
             elif job_type == "scrape_selected":
                 await self._scrape_channels(
-                    scraper, [str(channel_id) for channel_id in payload.get("channels", [])]
+                    scraper,
+                    [str(channel_id) for channel_id in payload.get("channels", [])],
                 )
             elif job_type == "export_all":
                 await scraper.export_data()

scripts/smoke_test.py

@@ -60,7 +60,11 @@ def main() -> int:
             status, body = fetch(endpoint)
             if status != 200:
                 raise RuntimeError(f"{endpoint} returned HTTP {status}")
-            if endpoint.endswith(".json") or endpoint.startswith("/api") or endpoint.startswith("/health"):
+            if (
+                endpoint.endswith(".json")
+                or endpoint.startswith("/api")
+                or endpoint.startswith("/health")
+            ):
                 json.loads(body.decode("utf-8"))
             print(f"ok {endpoint}")
         return 0

webui/index.html

@@ -4,7 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Telegram Scraper Control Panel</title>
-    <link rel="stylesheet" href="/static/style.css" />
+    <link rel="stylesheet" href="/static/style.css?v=2" />
   </head>
   <body>
     <div class="app-shell">

webui/style.css

@@ -47,7 +47,8 @@ button {
 }
 
 .app-shell {
-  min-height: 100vh;
+  height: 100vh;
+  overflow: hidden;
   display: grid;
   grid-template-columns: 300px minmax(0, 1fr);
 }
@@ -59,6 +60,7 @@ button {
 }
 
 .content {
+  overflow-y: auto;
   padding: 28px;
   max-width: 1320px;
   width: 100%;
@@ -494,6 +496,10 @@ tbody tr:hover {
   overflow-y: auto;
 }
 
+.sidebar-toggle {
+  display: none;
+}
+
 .viewer-sidebar-head {
   display: flex;
   align-items: flex-start;
@@ -814,6 +820,10 @@ tbody tr:hover {
     grid-template-columns: 1fr;
   }
 
+  .app-shell {
+    grid-template-rows: auto 1fr;
+  }
+
   .sidebar,
   .viewer-sidebar {
     border-right: 0;
@@ -825,14 +835,114 @@ tbody tr:hover {
   }
 }
 
-@media (max-width: 720px) {
+/* ---------- Mobile: tablet & phone ---------- */
-  .content,
+
-  .viewer-main,
+@media (max-width: 768px) {
-  .sidebar,
+  .sidebar-toggle {
+    display: inline-flex;
+  }
+
   .viewer-sidebar {
+    position: fixed;
+    left: -300px;
+    top: 0;
+    bottom: 0;
+    width: 280px;
+    z-index: 100;
+    transition: left 0.2s ease;
+    border-right: 1px dashed var(--line);
+    border-bottom: 0;
+  }
+
+  .viewer-sidebar.open {
+    left: 0;
+  }
+
+  .viewer-sidebar.open::before {
+    content: "";
+    position: fixed;
+    inset: 0;
+    background: rgba(0, 0, 0, 0.5);
+    z-index: -1;
+  }
+
+  .viewer-header {
+    padding: 12px 14px;
+  }
+
+  .messages-list {
+    padding: 8px 12px 16px;
+  }
+
+  .chat-bubble {
+    max-width: 92%;
+  }
+
+  .chat-bubble::before {
+    display: none;
+  }
+
+  .content {
     padding: 18px;
   }
 
+  .viewer-sidebar-head h1 {
+    font-size: 1.2rem;
+  }
+}
+
+@media (max-width: 480px) {
+  .content {
+    padding: 12px;
+  }
+
+  .viewer-main {
+    padding: 0;
+  }
+
+  .viewer-header {
+    padding: 10px 10px;
+  }
+
+  .viewer-header h2 {
+    font-size: 1.1rem;
+  }
+
+  .messages-list {
+    padding: 6px 8px 12px;
+  }
+
+  .chat-bubble {
+    max-width: 96%;
+    padding: 6px 9px;
+    font-size: 0.92rem;
+  }
+
+  .chat-footer {
+    font-size: 0.65rem;
+  }
+
+  .chat-sender {
+    font-size: 0.75rem;
+  }
+
+  .chat-reply-author {
+    font-size: 0.72rem;
+  }
+
+  .chat-reply-text {
+    font-size: 0.75rem;
+  }
+
+  .sidebar {
+    padding: 16px;
+  }
+
+  .viewer-sidebar {
+    width: 260px;
+    padding: 16px;
+  }
+
   .panel-header,
   .viewer-header,
   .dialog-header {

webui/viewer.html

@@ -4,7 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Telegram Scraper Viewer</title>
-    <link rel="stylesheet" href="/static/style.css" />
+    <link rel="stylesheet" href="/static/style.css?v=2" />
   </head>
   <body>
     <div class="viewer-shell">
@@ -14,6 +14,7 @@
             <div class="eyebrow">Export Viewer</div>
             <h1>Messages</h1>
           </div>
+          <button id="sidebar-toggle" class="button button-small sidebar-toggle">☰</button>
           <a class="button button-small" href="/">Dashboard</a>
         </div>
         <div id="viewer-channel-list" class="viewer-channel-list"></div>

webui/viewer.js

@@ -81,6 +81,7 @@ function renderChannelList() {
       node.classList.add("active");
     }
     node.addEventListener("click", () => {
+      document.querySelector(".viewer-sidebar")?.classList.remove("open");
       viewerState.oldestMessageId = null;
       viewerState.newestMessageId = null;
       loadChannel(channel.channel_id);
@@ -304,6 +305,19 @@ async function main() {
   }
 
   setupInfiniteScroll();
+
+  const toggle = document.getElementById("sidebar-toggle");
+  const sidebar = document.querySelector(".viewer-sidebar");
+  if (toggle && sidebar) {
+    toggle.addEventListener("click", () => {
+      sidebar.classList.toggle("open");
+    });
+    sidebar.addEventListener("click", (e) => {
+      if (e.target === sidebar) {
+        sidebar.classList.remove("open");
+      }
+    });
+  }
 }
 
 main().catch((error) => {

webui_server.py

@@ -306,7 +306,9 @@ class JobRunner:
                 if job.status == "running":
                     running_jobs.append(job)
         if running_jobs:
-            logger.warning("Waiting for %d running job(s) to finish...", len(running_jobs))
+            logger.warning(
+                "Waiting for %d running job(s) to finish...", len(running_jobs)
+            )
             deadline = time.time() + timeout
             while time.time() < deadline:
                 with self.lock:
@@ -561,6 +563,7 @@ class TelegramAuthManager:
         async def _disconnect():
             if self.client:
                 await self.client.disconnect()
+
         if self.client:
             try:
                 future = asyncio.run_coroutine_threadsafe(_disconnect(), self.loop)
@@ -831,7 +834,10 @@ def openapi_payload() -> Dict[str, Any]:
                     "requestBody": json_body(
                         {
                             "api_id": {"type": "integer", "example": 123456},
-                            "api_hash": {"type": "string", "example": "0123456789abcdef"},
+                            "api_hash": {
+                                "type": "string",
+                                "example": "0123456789abcdef",
+                            },
                         },
                         ["api_id", "api_hash"],
                     ),
@@ -929,7 +935,11 @@ def openapi_payload() -> Dict[str, Any]:
                         {
                             "name": "limit",
                             "in": "query",
-                            "schema": {"type": "integer", "default": 120, "maximum": 300},
+                            "schema": {
+                                "type": "integer",
+                                "default": 120,
+                                "maximum": 300,
+                            },
                         },
                         {
                             "name": "before",
@@ -945,7 +955,10 @@ def openapi_payload() -> Dict[str, Any]:
                     "summary": "Track a channel",
                     "requestBody": json_body(
                         {
-                            "channel_id": {"type": "string", "example": "-1001234567890"},
+                            "channel_id": {
+                                "type": "string",
+                                "example": "-1001234567890",
+                            },
                             "name": {"type": "string", "example": "Research feed"},
                         },
                         ["channel_id"],
@@ -990,7 +1003,10 @@ def openapi_payload() -> Dict[str, Any]:
                             "schema": {"type": "string"},
                         }
                     ],
-                    "responses": {**json_response, "404": {"description": "Job not found"}},
+                    "responses": {
+                        **json_response,
+                        "404": {"description": "Job not found"},
+                    },
                 }
             },
             "/api/jobs/scrape": {
@@ -1416,15 +1432,18 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             try:
                 start, end = self._parse_range(range_header, file_size)
             except ValueError:
-                self.send_error_json(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range")
+                self.send_error_json(
+                    HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range"
+                )
                 return
             content_length = end - start + 1
             self.send_response(HTTPStatus.PARTIAL_CONTENT)
             self.send_header("Content-Type", content_type)
             self.send_header("Accept-Ranges", "bytes")
-            self.send_header("Last-Modified", time.strftime(
+            self.send_header(
-                "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
+                "Last-Modified",
-            ))
+                time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
+            )
             self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
             self.send_header("Content-Length", str(content_length))
             self.end_headers()
@@ -1434,9 +1453,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             self.send_response(HTTPStatus.OK)
             self.send_header("Content-Type", content_type)
             self.send_header("Accept-Ranges", "bytes")
-            self.send_header("Last-Modified", time.strftime(
+            self.send_header(
-                "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)
+                "Last-Modified",
-            ))
+                time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
+            )
             self.send_header("Content-Length", str(file_size))
             self.end_headers()
             if not head_only:
@@ -1495,7 +1515,9 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
         self.job_runner = JobRunner()
         self.auth_manager = TelegramAuthManager()
         self.continuous_manager = ContinuousScrapeManager()
-        if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get("enabled", True):
+        if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get(
+            "enabled", True
+        ):
             self.continuous_manager.start()