fix(server): close residual review findings F-5..F-8
- Continuous endpoints reject non-list channels (400), null no longer wipes stored list (F-5) - '@'-prefixed legacy channel values normalized on load/save in manager and store (F-6) - _run_loop guarded: unexpected exceptions logged, backoff retry, no silent thread death (F-7) - +8 tests: continuous validation, normalization, loop survival, join-timeout/tombstone (F-8) - 58 tests passing; REVIEW.md updated
This commit is contained in:
@@ -1244,6 +1244,221 @@ class TestContinuousRestartDuringDrain:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
|
||||
class TestRound4ResidualFixes:
|
||||
"""F-5/F-6/F-7/F-8: residual round-3 findings fixed in round 4."""
|
||||
|
||||
def _setup_ws_data_dir(self):
|
||||
import webui_server as ws_module
|
||||
self._ws_orig_data = ws_module.DATA_DIR
|
||||
self._ws_orig_session = ws_module.SESSION_DIR
|
||||
ws_module.DATA_DIR = TEST_DATA
|
||||
ws_module.SESSION_DIR = TEST_SESSION
|
||||
ws_module.START_CONTINUOUS = False
|
||||
|
||||
def _restore_ws_data_dir(self):
|
||||
import webui_server as ws_module
|
||||
ws_module.DATA_DIR = self._ws_orig_data
|
||||
ws_module.SESSION_DIR = self._ws_orig_session
|
||||
|
||||
def _make_continuous_handler(self, account_id):
|
||||
"""Bare handler for _handle_account_continuous with a mocked server.
|
||||
|
||||
``handler.app`` is a property returning ``self.server``, so the mock
|
||||
app object (with ``continuous_orchestrator``) is installed as server.
|
||||
"""
|
||||
import webui_server as ws_module
|
||||
handler = object.__new__(ws_module.TelegramScraperRequestHandler)
|
||||
handler.account_id = account_id
|
||||
mock_app = MagicMock()
|
||||
mock_app.continuous_orchestrator.update_for = MagicMock(
|
||||
return_value={"config": {"enabled": True}, "status": {"running": False}}
|
||||
)
|
||||
handler.server = mock_app
|
||||
handler.send_error_json = MagicMock()
|
||||
handler.send_json = MagicMock()
|
||||
return handler
|
||||
|
||||
# ── F-5: `channels` present but not a list → 400, no silent wipe ──────
|
||||
|
||||
def test_handle_account_continuous_rejects_non_list_channels(self):
|
||||
"""F-5: non-list `channels` (string/int) must 400, not wipe the list."""
|
||||
import webui_server as ws_module
|
||||
self._setup_ws_data_dir()
|
||||
try:
|
||||
aid = make_account_id()
|
||||
create_account(TEST_DATA, aid, continuous_scraping={
|
||||
"enabled": True, "interval_minutes": 1, "channels": ["keep-me"],
|
||||
"run_all_tracked": True,
|
||||
})
|
||||
for bad in ("not-a-list", 42, {"a": 1}):
|
||||
handler = self._make_continuous_handler(aid)
|
||||
handler._handle_account_continuous(aid, {"enabled": True, "channels": bad})
|
||||
handler.send_error_json.assert_called_once()
|
||||
assert handler.send_error_json.call_args[0][0] == 400
|
||||
handler.app.continuous_orchestrator.update_for.assert_not_called()
|
||||
# Stored list must be untouched.
|
||||
stored = ws_module.load_account(TEST_DATA, aid)["continuous_scraping"]["channels"]
|
||||
assert stored == ["keep-me"]
|
||||
finally:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
def test_handle_account_continuous_null_channels_keeps_existing(self):
|
||||
"""F-5: `{"channels": null}` must keep the stored list, not wipe it."""
|
||||
self._setup_ws_data_dir()
|
||||
try:
|
||||
aid = make_account_id()
|
||||
create_account(TEST_DATA, aid, continuous_scraping={
|
||||
"enabled": True, "interval_minutes": 1, "channels": ["keep-me"],
|
||||
"run_all_tracked": True,
|
||||
})
|
||||
handler = self._make_continuous_handler(aid)
|
||||
handler._handle_account_continuous(aid, {"enabled": True, "channels": None})
|
||||
handler.send_error_json.assert_not_called()
|
||||
_, kwargs = handler.app.continuous_orchestrator.update_for.call_args
|
||||
assert kwargs["channels"] == ["keep-me"]
|
||||
finally:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
def test_handle_account_continuous_absent_channels_keeps_existing(self):
|
||||
"""F-5: key absent → read existing list from disk instead of `[]`."""
|
||||
self._setup_ws_data_dir()
|
||||
try:
|
||||
aid = make_account_id()
|
||||
create_account(TEST_DATA, aid, continuous_scraping={
|
||||
"enabled": True, "interval_minutes": 1, "channels": ["existing-1", "existing-2"],
|
||||
"run_all_tracked": True,
|
||||
})
|
||||
handler = self._make_continuous_handler(aid)
|
||||
handler._handle_account_continuous(aid, {"enabled": False})
|
||||
handler.send_error_json.assert_not_called()
|
||||
_, kwargs = handler.app.continuous_orchestrator.update_for.call_args
|
||||
assert kwargs["channels"] == ["existing-1", "existing-2"]
|
||||
finally:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
def test_handle_account_continuous_empty_list_still_clears(self):
|
||||
"""F-5: `channels: []` remains a valid, explicit clear."""
|
||||
self._setup_ws_data_dir()
|
||||
try:
|
||||
aid = make_account_id()
|
||||
create_account(TEST_DATA, aid, continuous_scraping={
|
||||
"enabled": True, "interval_minutes": 1, "channels": ["keep-me"],
|
||||
"run_all_tracked": True,
|
||||
})
|
||||
handler = self._make_continuous_handler(aid)
|
||||
handler._handle_account_continuous(aid, {"enabled": True, "channels": []})
|
||||
handler.send_error_json.assert_not_called()
|
||||
_, kwargs = handler.app.continuous_orchestrator.update_for.call_args
|
||||
assert kwargs["channels"] == []
|
||||
finally:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
# ── F-6: `@`-prefixed channels normalized on disk load ────────────────
|
||||
|
||||
def test_at_prefixed_channels_normalized_on_manager_load(self):
|
||||
"""F-6: manager must normalize `@channel` values read from disk."""
|
||||
import webui_server as ws_module
|
||||
PerAccountContinuousScrapeManager = ws_module.PerAccountContinuousScrapeManager
|
||||
self._setup_ws_data_dir()
|
||||
try:
|
||||
aid = make_account_id()
|
||||
create_account(TEST_DATA, aid, continuous_scraping={
|
||||
"enabled": True,
|
||||
"interval_minutes": 1,
|
||||
"channels": ["@with_at", " @padded_at ", "plain", -100123],
|
||||
"run_all_tracked": True,
|
||||
})
|
||||
mgr = PerAccountContinuousScrapeManager(aid)
|
||||
# @-prefixes and padding stripped; numbers kept; no @ remains.
|
||||
assert mgr.config["channels"] == ["with_at", "padded_at", "plain", "-100123"]
|
||||
# refresh_config must apply the same normalization.
|
||||
mgr.refresh_config()
|
||||
assert mgr.config["channels"] == ["with_at", "padded_at", "plain", "-100123"]
|
||||
finally:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
# ── F-7: top-level exception in _run_loop must not kill the worker ─────
|
||||
|
||||
def test_run_loop_survives_top_level_exception(self):
|
||||
"""F-7: an exception outside the scrape iteration sets last_error and
|
||||
exits cleanly (loop breaks on stop) instead of dying silently."""
|
||||
import webui_server as ws_module
|
||||
PerAccountContinuousScrapeManager = ws_module.PerAccountContinuousScrapeManager
|
||||
self._setup_ws_data_dir()
|
||||
try:
|
||||
aid = make_account_id()
|
||||
create_account(TEST_DATA, aid, continuous_scraping={
|
||||
"enabled": True, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
|
||||
})
|
||||
mgr = PerAccountContinuousScrapeManager(aid)
|
||||
|
||||
def boom():
|
||||
mgr.stop_event.set() # wake the post-error backoff immediately
|
||||
raise RuntimeError("boom outside iteration")
|
||||
|
||||
mgr.refresh_config = boom
|
||||
original_exception = ws_module.logger.exception
|
||||
ws_module.logger.exception = lambda *a, **k: None # keep test output clean
|
||||
try:
|
||||
mgr.status["running"] = True # mimic a started worker
|
||||
mgr._run_loop() # synchronous, deterministic
|
||||
finally:
|
||||
ws_module.logger.exception = original_exception
|
||||
assert mgr.status["last_error"] == "Unexpected loop error (see logs)"
|
||||
# finally block ran: thread truly exited.
|
||||
assert mgr.status["running"] is False
|
||||
finally:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
# ── F-8: join() → False on real timeout, plus shutdown-guard ───────────
|
||||
|
||||
def test_join_returns_false_when_thread_does_not_exit_in_time(self):
|
||||
"""F-8: join() must report False when the worker does not exit within
|
||||
the timeout, and True once it actually finishes."""
|
||||
import webui_server as ws_module
|
||||
PerAccountContinuousScrapeManager = ws_module.PerAccountContinuousScrapeManager
|
||||
self._setup_ws_data_dir()
|
||||
try:
|
||||
aid = make_account_id()
|
||||
create_account(TEST_DATA, aid, continuous_scraping={
|
||||
"enabled": True, "interval_minutes": 1, "channels": [], "run_all_tracked": True,
|
||||
})
|
||||
mgr = PerAccountContinuousScrapeManager(aid)
|
||||
sleeper = threading.Thread(target=time.sleep, args=(1.0,))
|
||||
sleeper.start()
|
||||
mgr.thread = sleeper
|
||||
try:
|
||||
assert mgr.join(timeout=0.05) is False, "thread still alive → False"
|
||||
assert mgr.join(timeout=5.0) is True, "thread finished → True"
|
||||
finally:
|
||||
if sleeper.is_alive():
|
||||
sleeper.join(timeout=5.0)
|
||||
finally:
|
||||
self._restore_ws_data_dir()
|
||||
|
||||
def test_job_runner_rejects_create_job_during_shutdown(self):
|
||||
"""M-7/F-8: create_job must reject new jobs once shutdown has begun,
|
||||
and accept them again only after the flag is cleared."""
|
||||
import webui_server as ws_module
|
||||
runner = ws_module.JobRunner()
|
||||
runner.queue.put = lambda job: None
|
||||
try:
|
||||
runner._shutdown_flag = True
|
||||
raised = False
|
||||
try:
|
||||
runner.create_job("scrape_all", "During shutdown", {"account_id": "shut-acc"})
|
||||
except RuntimeError:
|
||||
raised = True
|
||||
assert raised, "create_job during shutdown must raise RuntimeError"
|
||||
|
||||
runner._shutdown_flag = False
|
||||
job = runner.create_job("scrape_all", "After shutdown flag cleared", {"account_id": "ok-acc"})
|
||||
assert job is not None and job.job_type == "scrape_all"
|
||||
finally:
|
||||
runner._shutdown_flag = False
|
||||
runner.shutdown(timeout=0)
|
||||
|
||||
|
||||
# ── Cleanup all temp data ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user