65 lines
2.8 KiB
Python
65 lines
2.8 KiB
Python
# agent-office/tests/test_node_monitor.py
|
|
import json, pytest
|
|
from app import node_monitor
|
|
|
|
class FakeRedis:
|
|
"""worker heartbeat + queue llen + scan_iter 흉내."""
|
|
def __init__(self, kv=None, lists=None):
|
|
self._kv = kv or {} # key(str) -> bytes
|
|
self._lists = lists or {} # key(str) -> length(int)
|
|
async def get(self, key):
|
|
return self._kv.get(key)
|
|
async def llen(self, key):
|
|
return self._lists.get(key, 0)
|
|
async def scan_iter(self, match=None):
|
|
prefix = match.rstrip("*")
|
|
for k in list(self._lists):
|
|
if k.startswith(prefix):
|
|
yield k
|
|
|
|
def _hb(name, kind, state, **extra):
|
|
return json.dumps({"name": name, "kind": kind, "state": state, "ts": "2026-06-29T00:00:00Z",
|
|
"last_job_at": None, "jobs_done": 0, "jobs_failed": 0, **extra}).encode()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_alive_worker_healthy_link():
|
|
r = FakeRedis(kv={"worker:image-render:heartbeat": _hb("image-render","render","idle")})
|
|
st = await node_monitor.collect_status(redis=r)
|
|
img = next(w for w in st["workers"] if w["name"] == "image-render")
|
|
assert img["alive"] is True and img["state"] == "idle"
|
|
link = next(l for l in st["links"] if l["to"] == "image-render")
|
|
assert link["status"] == "healthy" and link["type"] == "redis-queue"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_missing_heartbeat_is_dead_and_down():
|
|
r = FakeRedis() # heartbeat 없음
|
|
st = await node_monitor.collect_status(redis=r)
|
|
img = next(w for w in st["workers"] if w["name"] == "image-render")
|
|
assert img["alive"] is False
|
|
link = next(l for l in st["links"] if l["to"] == "image-render")
|
|
assert link["status"] == "down"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dead_letter_makes_degraded():
|
|
r = FakeRedis(kv={"worker:video-render:heartbeat": _hb("video-render","render","idle")},
|
|
lists={"dead_letter:queue:video-render": 2})
|
|
st = await node_monitor.collect_status(redis=r)
|
|
vid = next(w for w in st["workers"] if w["name"] == "video-render")
|
|
assert vid["dead_letter"] == 2
|
|
link = next(l for l in st["links"] if l["to"] == "video-render")
|
|
assert link["status"] == "degraded"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_paused_reason_from_watcher():
|
|
r = FakeRedis(kv={"queue:paused": b"1",
|
|
"worker:task-watcher:heartbeat": _hb("task-watcher","watcher","trading",mode="trading")})
|
|
st = await node_monitor.collect_status(redis=r)
|
|
assert st["paused"] is True and st["paused_reason"] == "trading"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_trader_http_pull_link():
|
|
r = FakeRedis(kv={"worker:ai_trade:heartbeat": _hb("ai_trade","trader","market_open")})
|
|
st = await node_monitor.collect_status(redis=r)
|
|
link = next(l for l in st["links"] if l["from"] == "ai_trade")
|
|
assert link["type"] == "http-pull" and link["status"] == "healthy"
|