Harden scraper inputs and exports

.gitea/workflows/ci.yaml

@@ -124,7 +124,7 @@ jobs:
 
   publish:
     needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
-    if: github.event_name != 'pull_request'
+    if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository

.github/workflows/ci.yaml

@@ -124,7 +124,7 @@ jobs:
 
   publish:
     needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
-    if: github.event_name != 'pull_request'
+    if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
     runs-on: [self-hosted, linux, arch, homelab]
     steps:
       - name: Checkout repository

pyproject.toml

@@ -20,3 +20,6 @@ dependencies = [
     "Telethon==1.40.0",
     "yarl==1.20.1",
 ]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]

tests/test_integration.py

@@ -284,6 +284,34 @@ class TestAccountLifecycle:
         assert loaded["continuous_scraping"]["run_all_tracked"] is True
 
 
+class TestInputSafety:
+    """Small safety checks for values that flow into files or exports."""
+
+    def test_channel_id_rejects_path_characters(self):
+        import webui_server as ws_module
+
+        assert ws_module.normalize_channel_id("@valid_name") == "valid_name"
+        for bad in ("../escape", "nested/channel", r"nested\channel", ".", "..", ""):
+            try:
+                ws_module.normalize_channel_id(bad)
+            except ValueError:
+                continue
+            raise AssertionError(f"accepted unsafe channel_id: {bad!r}")
+
+    def test_export_account_state_redacts_api_hash_by_default(self):
+        import webui_server as ws_module
+
+        state = {"label": "Work", "api_id": 123, "api_hash": "secret", "channels": {}}
+        redacted = ws_module.export_account_state(state)
+        assert redacted["api_hash"] is None
+        assert redacted["api_hash_present"] is True
+        assert state["api_hash"] == "secret"
+
+        full = ws_module.export_account_state(state, include_secrets=True)
+        assert full["api_hash"] == "secret"
+        assert "api_hash_present" not in full
+
+
 class TestContinuousOrchestrator:
     """Continuous scraping manager safety using mocked webui_server."""
 
@@ -536,27 +564,33 @@ class TestJobDeduplicationSchema:
     def test_create_job_dedup_by_account(self):
         JobRunner = self._import_job_runner()
         runner = JobRunner()
+        runner.queue.put = lambda job: None
         j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
         j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
         assert j1.job_id == j2.job_id, "duplicate job was created instead of being reused"
         assert "Reused existing active job" in j2.logs
+        runner.shutdown(timeout=0)
 
     def test_create_job_allows_different_accounts(self):
         JobRunner = self._import_job_runner()
         runner = JobRunner()
+        runner.queue.put = lambda job: None
         j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
         time.sleep(0.002)  # ensure different timestamp -> different job_id
         j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc2"})
         assert j1.job_id != j2.job_id
+        runner.shutdown(timeout=0)
 
     def test_create_job_allows_after_previous_completes(self):
         JobRunner = self._import_job_runner()
         runner = JobRunner()
+        runner.queue.put = lambda job: None
         j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"})
-        j1.status = "completed"
+        j1.status = "done"
         time.sleep(0.002)  # ensure different timestamp -> different job_id
         j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"})
         assert j1.job_id != j2.job_id
+        runner.shutdown(timeout=0)
 
 
 class TestAccountHealthSummary:

webui_server.py

@@ -99,6 +99,27 @@ def normalize_media_url(media_path: Optional[str]) -> Optional[str]:
     return "/media/" + urllib.parse.quote(str(relative).replace("\\", "/"))
 
 
+def normalize_channel_id(value: Any) -> str:
+    channel_id = str(value or "").strip().lstrip("@")
+    if (
+        not channel_id
+        or "/" in channel_id
+        or "\\" in channel_id
+        or channel_id in {".", ".."}
+        or any(ord(ch) < 32 for ch in channel_id)
+    ):
+        raise ValueError("channel_id contains unsupported path characters")
+    return channel_id
+
+
+def export_account_state(state: Dict[str, Any], include_secrets: bool = False) -> Dict[str, Any]:
+    exported = dict(state)
+    if not include_secrets and "api_hash" in exported:
+        exported["api_hash_present"] = bool(exported.get("api_hash"))
+        exported["api_hash"] = None
+    return exported
+
+
 def channel_db_path(account_id: Optional[str], channel_id: str) -> Path:
     if account_id:
         return account_data_dir(DATA_DIR, account_id) / channel_id / f"{channel_id}.db"
@@ -1832,7 +1853,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_json(list_channels_snapshot())
         if path.startswith("/api/channels/") and path.endswith("/messages"):
             parts = path.split("/")
-            channel_id = urllib.parse.unquote(parts[3]).lstrip("@")
+            try:
+                channel_id = normalize_channel_id(urllib.parse.unquote(parts[3]))
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             limit = max(1, min(int(query.get("limit", ["120"])[0]), 300))
             before = query.get("before")
             before_message_id = int(before[0]) if before else None
@@ -1912,16 +1936,20 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
         if sub == ["health"]:
             return self.send_json(account_health_summary(account_id, self.app.job_runner))
         if sub == ["export"]:
+            include_secrets = query.get("include_secrets", ["0"])[0].lower() in {"1", "true", "yes"}
             payload = {
                 "version": 1,
                 "account_id": account_id,
-                "state": load_account(DATA_DIR, account_id),
+                "state": export_account_state(load_account(DATA_DIR, account_id), include_secrets=include_secrets),
             }
             return self.send_json(payload)
         if sub == ["channels"]:
             return self._handle_get_account_channels(account_id)
         if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages":
-            channel_id = urllib.parse.unquote(sub[1]).lstrip("@")
+            try:
+                channel_id = normalize_channel_id(urllib.parse.unquote(sub[1]))
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             return self._handle_get_account_channel_messages(account_id, channel_id, query)
         if sub == ["continuous"]:
             return self.send_json(
@@ -2117,9 +2145,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_error_json(HTTPStatus.BAD_REQUEST, "Legacy auth is not available in multi-account mode. Use /api/accounts/{id}/auth/password instead.")
 
         if path == "/api/channels/add":
-            channel_id = str(body.get("channel_id", "")).strip()
-            if not channel_id:
-                return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+            try:
+                channel_id = normalize_channel_id(body.get("channel_id"))
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             if self.app.legacy_account_id:
                 return self._handle_account_channel_add(self.app.legacy_account_id, channel_id, body.get("name"))
             state = load_state()
@@ -2131,7 +2160,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_json({"ok": True, "channel_id": channel_id})
 
         if path == "/api/channels/remove":
-            channel_id = str(body.get("channel_id", "")).strip()
+            try:
+                channel_id = normalize_channel_id(body.get("channel_id"))
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             if self.app.legacy_account_id:
                 return self._handle_account_channel_remove(self.app.legacy_account_id, channel_id)
             state = load_state()
@@ -2147,10 +2179,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             if account_id:
                 payload["account_id"] = account_id
             if channel_id:
-                payload["channel_id"] = str(channel_id)
+                try:
+                    payload["channel_id"] = normalize_channel_id(channel_id)
+                except ValueError as exc:
+                    return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
                 job = self.app.job_runner.create_job(
                     "scrape_channel",
-                    f"Scrape channel {channel_id}" + (f" [{account_id}]" if account_id else ""),
+                    f"Scrape channel {payload['channel_id']}" + (f" [{account_id}]" if account_id else ""),
                     payload,
                 )
             else:
@@ -2173,9 +2208,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
 
         if path == "/api/jobs/export-channel":
-            channel_id = str(body.get("channel_id", "")).strip()
-            if not channel_id:
-                return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+            try:
+                channel_id = normalize_channel_id(body.get("channel_id"))
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             payload = {"channel_id": channel_id}
             if self.app.legacy_account_id:
                 payload["account_id"] = self.app.legacy_account_id
@@ -2187,9 +2223,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
 
         if path == "/api/jobs/rescrape-media":
-            channel_id = str(body.get("channel_id", "")).strip()
-            if not channel_id:
-                return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+            try:
+                channel_id = normalize_channel_id(body.get("channel_id"))
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             payload = {"channel_id": channel_id}
             if self.app.legacy_account_id:
                 payload["account_id"] = self.app.legacy_account_id
@@ -2201,9 +2238,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
 
         if path == "/api/jobs/fix-media":
-            channel_id = str(body.get("channel_id", "")).strip()
-            if not channel_id:
-                return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+            try:
+                channel_id = normalize_channel_id(body.get("channel_id"))
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             payload = {"channel_id": channel_id}
             if self.app.legacy_account_id:
                 payload["account_id"] = self.app.legacy_account_id
@@ -2293,8 +2331,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             "label": str(body.get("label") or state.get("label") or account_id).strip(),
             "api_id": state.get("api_id"),
             "api_hash": state.get("api_hash"),
-            "channels": state.get("channels") if isinstance(state.get("channels"), dict) else {},
-            "channel_names": state.get("channel_names") if isinstance(state.get("channel_names"), dict) else {},
+            "channels": self._clean_imported_channels(state.get("channels")),
+            "channel_names": self._clean_imported_channel_names(state.get("channel_names")),
             "scrape_media": bool(state.get("scrape_media", True)),
             "forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [],
             "continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else {
@@ -2337,9 +2375,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
         if sub == ["auth", "password"]:
             return self._handle_post_account_auth_password(account_id, body)
         if sub == ["channels", "add"]:
-            return self._handle_account_channel_add(account_id, str(body.get("channel_id", "")).strip(), body.get("name"))
+            return self._handle_account_channel_add(account_id, body.get("channel_id"), body.get("name"))
         if sub == ["channels", "remove"]:
-            return self._handle_account_channel_remove(account_id, str(body.get("channel_id", "")).strip())
+            return self._handle_account_channel_remove(account_id, body.get("channel_id"))
         if sub == ["jobs", "scrape"]:
             return self._handle_account_job_scrape(account_id, body)
         if sub == ["jobs", "export"]:
@@ -2412,10 +2450,35 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
         return self.send_json(payload)
 
-    def _handle_account_channel_add(self, account_id: str, channel_id: str, name: Optional[str]) -> None:
-        channel_id = channel_id.lstrip("@")
-        if not channel_id:
-            return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+    def _clean_imported_channels(self, channels: Any) -> Dict[str, Any]:
+        if not isinstance(channels, dict):
+            return {}
+        cleaned: Dict[str, Any] = {}
+        for raw_channel_id, last_message_id in channels.items():
+            try:
+                channel_id = normalize_channel_id(raw_channel_id)
+            except ValueError:
+                continue
+            cleaned[channel_id] = last_message_id
+        return cleaned
+
+    def _clean_imported_channel_names(self, channel_names: Any) -> Dict[str, str]:
+        if not isinstance(channel_names, dict):
+            return {}
+        cleaned: Dict[str, str] = {}
+        for raw_channel_id, name in channel_names.items():
+            try:
+                channel_id = normalize_channel_id(raw_channel_id)
+            except ValueError:
+                continue
+            cleaned[channel_id] = str(name).strip()
+        return cleaned
+
+    def _handle_account_channel_add(self, account_id: str, channel_id: Any, name: Optional[str]) -> None:
+        try:
+            channel_id = normalize_channel_id(channel_id)
+        except ValueError as exc:
+            return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
         store = get_account_store(DATA_DIR, account_id)
         def mutate(state: Dict[str, Any]) -> None:
             if channel_id not in state.setdefault("channels", {}):
@@ -2426,9 +2489,11 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
         self.app.continuous_orchestrator.refresh_account(account_id)
         return self.send_json({"ok": True, "channel_id": channel_id})
 
-    def _handle_account_channel_remove(self, account_id: str, channel_id: str) -> None:
-        if not channel_id:
-            return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+    def _handle_account_channel_remove(self, account_id: str, channel_id: Any) -> None:
+        try:
+            channel_id = normalize_channel_id(channel_id)
+        except ValueError as exc:
+            return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
         store = get_account_store(DATA_DIR, account_id)
         existed = [False]
         def mutate(state: Dict[str, Any]) -> None:
@@ -2444,6 +2509,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
     def _handle_account_job_scrape(self, account_id: str, body: Dict[str, Any]) -> None:
         channel_id = body.get("channel_id")
         if channel_id:
+            try:
+                channel_id = normalize_channel_id(channel_id)
+            except ValueError as exc:
+                return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
             job = self.app.job_runner.create_job(
                 "scrape_channel",
                 f"Scrape channel {channel_id} [{account_id}]",
@@ -2466,9 +2535,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
         return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
 
     def _handle_account_job_export_channel(self, account_id: str, body: Dict[str, Any]) -> None:
-        channel_id = str(body.get("channel_id", "")).strip()
-        if not channel_id:
-            return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+        try:
+            channel_id = normalize_channel_id(body.get("channel_id"))
+        except ValueError as exc:
+            return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
         job = self.app.job_runner.create_job(
             "export_channel",
             f"Export channel {channel_id} [{account_id}]",
@@ -2477,9 +2547,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
         return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
 
     def _handle_account_job_rescrape_media(self, account_id: str, body: Dict[str, Any]) -> None:
-        channel_id = str(body.get("channel_id", "")).strip()
-        if not channel_id:
-            return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+        try:
+            channel_id = normalize_channel_id(body.get("channel_id"))
+        except ValueError as exc:
+            return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
         job = self.app.job_runner.create_job(
             "rescrape_media",
             f"Rescrape media for {channel_id} [{account_id}]",
@@ -2488,9 +2559,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
         return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED)
 
     def _handle_account_job_fix_media(self, account_id: str, body: Dict[str, Any]) -> None:
-        channel_id = str(body.get("channel_id", "")).strip()
-        if not channel_id:
-            return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required")
+        try:
+            channel_id = normalize_channel_id(body.get("channel_id"))
+        except ValueError as exc:
+            return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
         job = self.app.job_runner.create_job(
             "fix_missing_media",
             f"Fix missing media for {channel_id} [{account_id}]",