diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 77c9a55..3238cac 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -124,7 +124,7 @@ jobs: publish: needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate] - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev') runs-on: [self-hosted, linux, arch, homelab] steps: - name: Checkout repository diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 77c9a55..3238cac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -124,7 +124,7 @@ jobs: publish: needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate] - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev') runs-on: [self-hosted, linux, arch, homelab] steps: - name: Checkout repository diff --git a/pyproject.toml b/pyproject.toml index 12ea529..2cc075d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,6 @@ dependencies = [ "Telethon==1.40.0", "yarl==1.20.1", ] + +[tool.pytest.ini_options] +pythonpath = ["."] diff --git a/tests/test_integration.py b/tests/test_integration.py index c07ad83..17a7457 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -284,6 +284,34 @@ class TestAccountLifecycle: assert loaded["continuous_scraping"]["run_all_tracked"] is True +class TestInputSafety: + """Small safety checks for values that flow into files or exports.""" + + def test_channel_id_rejects_path_characters(self): + import webui_server as ws_module + + assert ws_module.normalize_channel_id("@valid_name") == "valid_name" + for bad in ("../escape", "nested/channel", r"nested\channel", ".", "..", ""): + try: + ws_module.normalize_channel_id(bad) + except ValueError: + continue + raise AssertionError(f"accepted unsafe channel_id: {bad!r}") + + def test_export_account_state_redacts_api_hash_by_default(self): + import webui_server as ws_module + + state = {"label": "Work", "api_id": 123, "api_hash": "secret", "channels": {}} + redacted = ws_module.export_account_state(state) + assert redacted["api_hash"] is None + assert redacted["api_hash_present"] is True + assert state["api_hash"] == "secret" + + full = ws_module.export_account_state(state, include_secrets=True) + assert full["api_hash"] == "secret" + assert "api_hash_present" not in full + + class TestContinuousOrchestrator: """Continuous scraping manager safety using mocked webui_server.""" @@ -536,27 +564,33 @@ class TestJobDeduplicationSchema: def test_create_job_dedup_by_account(self): JobRunner = self._import_job_runner() runner = JobRunner() + runner.queue.put = lambda job: None j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"}) j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"}) assert j1.job_id == j2.job_id, "duplicate job was created instead of being reused" assert "Reused existing active job" in j2.logs + runner.shutdown(timeout=0) def test_create_job_allows_different_accounts(self): JobRunner = self._import_job_runner() runner = JobRunner() + runner.queue.put = lambda job: None j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"}) time.sleep(0.002) # ensure different timestamp -> different job_id j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc2"}) assert j1.job_id != j2.job_id + runner.shutdown(timeout=0) def test_create_job_allows_after_previous_completes(self): JobRunner = self._import_job_runner() runner = JobRunner() + runner.queue.put = lambda job: None j1 = runner.create_job("scrape_all", "First", {"account_id": "acc1"}) - j1.status = "completed" + j1.status = "done" time.sleep(0.002) # ensure different timestamp -> different job_id j2 = runner.create_job("scrape_all", "Second", {"account_id": "acc1"}) assert j1.job_id != j2.job_id + runner.shutdown(timeout=0) class TestAccountHealthSummary: diff --git a/webui_server.py b/webui_server.py index f3b592d..d3b64aa 100644 --- a/webui_server.py +++ b/webui_server.py @@ -99,6 +99,27 @@ def normalize_media_url(media_path: Optional[str]) -> Optional[str]: return "/media/" + urllib.parse.quote(str(relative).replace("\\", "/")) +def normalize_channel_id(value: Any) -> str: + channel_id = str(value or "").strip().lstrip("@") + if ( + not channel_id + or "/" in channel_id + or "\\" in channel_id + or channel_id in {".", ".."} + or any(ord(ch) < 32 for ch in channel_id) + ): + raise ValueError("channel_id contains unsupported path characters") + return channel_id + + +def export_account_state(state: Dict[str, Any], include_secrets: bool = False) -> Dict[str, Any]: + exported = dict(state) + if not include_secrets and "api_hash" in exported: + exported["api_hash_present"] = bool(exported.get("api_hash")) + exported["api_hash"] = None + return exported + + def channel_db_path(account_id: Optional[str], channel_id: str) -> Path: if account_id: return account_data_dir(DATA_DIR, account_id) / channel_id / f"{channel_id}.db" @@ -1832,7 +1853,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(list_channels_snapshot()) if path.startswith("/api/channels/") and path.endswith("/messages"): parts = path.split("/") - channel_id = urllib.parse.unquote(parts[3]).lstrip("@") + try: + channel_id = normalize_channel_id(urllib.parse.unquote(parts[3])) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) limit = max(1, min(int(query.get("limit", ["120"])[0]), 300)) before = query.get("before") before_message_id = int(before[0]) if before else None @@ -1912,16 +1936,20 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if sub == ["health"]: return self.send_json(account_health_summary(account_id, self.app.job_runner)) if sub == ["export"]: + include_secrets = query.get("include_secrets", ["0"])[0].lower() in {"1", "true", "yes"} payload = { "version": 1, "account_id": account_id, - "state": load_account(DATA_DIR, account_id), + "state": export_account_state(load_account(DATA_DIR, account_id), include_secrets=include_secrets), } return self.send_json(payload) if sub == ["channels"]: return self._handle_get_account_channels(account_id) if len(sub) >= 3 and sub[0] == "channels" and sub[-1] == "messages": - channel_id = urllib.parse.unquote(sub[1]).lstrip("@") + try: + channel_id = normalize_channel_id(urllib.parse.unquote(sub[1])) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self._handle_get_account_channel_messages(account_id, channel_id, query) if sub == ["continuous"]: return self.send_json( @@ -2117,9 +2145,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_error_json(HTTPStatus.BAD_REQUEST, "Legacy auth is not available in multi-account mode. Use /api/accounts/{id}/auth/password instead.") if path == "/api/channels/add": - channel_id = str(body.get("channel_id", "")).strip() - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) if self.app.legacy_account_id: return self._handle_account_channel_add(self.app.legacy_account_id, channel_id, body.get("name")) state = load_state() @@ -2131,7 +2160,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json({"ok": True, "channel_id": channel_id}) if path == "/api/channels/remove": - channel_id = str(body.get("channel_id", "")).strip() + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) if self.app.legacy_account_id: return self._handle_account_channel_remove(self.app.legacy_account_id, channel_id) state = load_state() @@ -2147,10 +2179,13 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if account_id: payload["account_id"] = account_id if channel_id: - payload["channel_id"] = str(channel_id) + try: + payload["channel_id"] = normalize_channel_id(channel_id) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) job = self.app.job_runner.create_job( "scrape_channel", - f"Scrape channel {channel_id}" + (f" [{account_id}]" if account_id else ""), + f"Scrape channel {payload['channel_id']}" + (f" [{account_id}]" if account_id else ""), payload, ) else: @@ -2173,9 +2208,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) if path == "/api/jobs/export-channel": - channel_id = str(body.get("channel_id", "")).strip() - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) payload = {"channel_id": channel_id} if self.app.legacy_account_id: payload["account_id"] = self.app.legacy_account_id @@ -2187,9 +2223,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) if path == "/api/jobs/rescrape-media": - channel_id = str(body.get("channel_id", "")).strip() - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) payload = {"channel_id": channel_id} if self.app.legacy_account_id: payload["account_id"] = self.app.legacy_account_id @@ -2201,9 +2238,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) if path == "/api/jobs/fix-media": - channel_id = str(body.get("channel_id", "")).strip() - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) payload = {"channel_id": channel_id} if self.app.legacy_account_id: payload["account_id"] = self.app.legacy_account_id @@ -2293,8 +2331,8 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): "label": str(body.get("label") or state.get("label") or account_id).strip(), "api_id": state.get("api_id"), "api_hash": state.get("api_hash"), - "channels": state.get("channels") if isinstance(state.get("channels"), dict) else {}, - "channel_names": state.get("channel_names") if isinstance(state.get("channel_names"), dict) else {}, + "channels": self._clean_imported_channels(state.get("channels")), + "channel_names": self._clean_imported_channel_names(state.get("channel_names")), "scrape_media": bool(state.get("scrape_media", True)), "forwarding_rules": state.get("forwarding_rules") if isinstance(state.get("forwarding_rules"), list) else [], "continuous_scraping": state.get("continuous_scraping") if isinstance(state.get("continuous_scraping"), dict) else { @@ -2337,9 +2375,9 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): if sub == ["auth", "password"]: return self._handle_post_account_auth_password(account_id, body) if sub == ["channels", "add"]: - return self._handle_account_channel_add(account_id, str(body.get("channel_id", "")).strip(), body.get("name")) + return self._handle_account_channel_add(account_id, body.get("channel_id"), body.get("name")) if sub == ["channels", "remove"]: - return self._handle_account_channel_remove(account_id, str(body.get("channel_id", "")).strip()) + return self._handle_account_channel_remove(account_id, body.get("channel_id")) if sub == ["jobs", "scrape"]: return self._handle_account_job_scrape(account_id, body) if sub == ["jobs", "export"]: @@ -2412,10 +2450,35 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self.send_json(payload) - def _handle_account_channel_add(self, account_id: str, channel_id: str, name: Optional[str]) -> None: - channel_id = channel_id.lstrip("@") - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + def _clean_imported_channels(self, channels: Any) -> Dict[str, Any]: + if not isinstance(channels, dict): + return {} + cleaned: Dict[str, Any] = {} + for raw_channel_id, last_message_id in channels.items(): + try: + channel_id = normalize_channel_id(raw_channel_id) + except ValueError: + continue + cleaned[channel_id] = last_message_id + return cleaned + + def _clean_imported_channel_names(self, channel_names: Any) -> Dict[str, str]: + if not isinstance(channel_names, dict): + return {} + cleaned: Dict[str, str] = {} + for raw_channel_id, name in channel_names.items(): + try: + channel_id = normalize_channel_id(raw_channel_id) + except ValueError: + continue + cleaned[channel_id] = str(name).strip() + return cleaned + + def _handle_account_channel_add(self, account_id: str, channel_id: Any, name: Optional[str]) -> None: + try: + channel_id = normalize_channel_id(channel_id) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) store = get_account_store(DATA_DIR, account_id) def mutate(state: Dict[str, Any]) -> None: if channel_id not in state.setdefault("channels", {}): @@ -2426,9 +2489,11 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): self.app.continuous_orchestrator.refresh_account(account_id) return self.send_json({"ok": True, "channel_id": channel_id}) - def _handle_account_channel_remove(self, account_id: str, channel_id: str) -> None: - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + def _handle_account_channel_remove(self, account_id: str, channel_id: Any) -> None: + try: + channel_id = normalize_channel_id(channel_id) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) store = get_account_store(DATA_DIR, account_id) existed = [False] def mutate(state: Dict[str, Any]) -> None: @@ -2444,6 +2509,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): def _handle_account_job_scrape(self, account_id: str, body: Dict[str, Any]) -> None: channel_id = body.get("channel_id") if channel_id: + try: + channel_id = normalize_channel_id(channel_id) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) job = self.app.job_runner.create_job( "scrape_channel", f"Scrape channel {channel_id} [{account_id}]", @@ -2466,9 +2535,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) def _handle_account_job_export_channel(self, account_id: str, body: Dict[str, Any]) -> None: - channel_id = str(body.get("channel_id", "")).strip() - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) job = self.app.job_runner.create_job( "export_channel", f"Export channel {channel_id} [{account_id}]", @@ -2477,9 +2547,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) def _handle_account_job_rescrape_media(self, account_id: str, body: Dict[str, Any]) -> None: - channel_id = str(body.get("channel_id", "")).strip() - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) job = self.app.job_runner.create_job( "rescrape_media", f"Rescrape media for {channel_id} [{account_id}]", @@ -2488,9 +2559,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler): return self.send_json(job.to_dict(), status=HTTPStatus.ACCEPTED) def _handle_account_job_fix_media(self, account_id: str, body: Dict[str, Any]) -> None: - channel_id = str(body.get("channel_id", "")).strip() - if not channel_id: - return self.send_error_json(HTTPStatus.BAD_REQUEST, "channel_id is required") + try: + channel_id = normalize_channel_id(body.get("channel_id")) + except ValueError as exc: + return self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) job = self.app.job_runner.create_job( "fix_missing_media", f"Fix missing media for {channel_id} [{account_id}]",