feat(agent-office): 노드 헬스 1분 cron + 텔레그램 경보(다운/복구/dead-letter)
This commit is contained in:
@@ -96,3 +96,31 @@ async def test_llen_exception_returns_redis_ok_false():
|
||||
)
|
||||
st = await node_monitor.collect_status(redis=r)
|
||||
assert st["redis_ok"] is False
|
||||
|
||||
|
||||
import app.node_monitor as nm
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alert_on_alive_to_dead(monkeypatch):
|
||||
sent = []
|
||||
async def fake_send_raw(text, **kw): sent.append(text); return {"ok": True}
|
||||
monkeypatch.setattr("app.telegram.messaging.send_raw", fake_send_raw)
|
||||
monkeypatch.setattr("app.db.add_log", lambda *a, **k: None)
|
||||
nm._node_state.clear(); nm._dl_notified.clear()
|
||||
alive = {"workers": [{"name":"image-render","alive":True,"dead_letter":0}], "links": []}
|
||||
dead = {"workers": [{"name":"image-render","alive":False,"dead_letter":0}], "links": []}
|
||||
await nm.check_and_alert(status=alive) # 첫 관측 — 경보 없음
|
||||
assert sent == []
|
||||
await nm.check_and_alert(status=dead) # alive→dead 전이
|
||||
assert any("다운" in t for t in sent)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alert_on_dead_letter_growth(monkeypatch):
|
||||
sent = []
|
||||
async def fake_send_raw(text, **kw): sent.append(text); return {"ok": True}
|
||||
monkeypatch.setattr("app.telegram.messaging.send_raw", fake_send_raw)
|
||||
monkeypatch.setattr("app.db.add_log", lambda *a, **k: None)
|
||||
nm._node_state.clear(); nm._dl_notified.clear()
|
||||
s = {"workers": [{"name":"video-render","alive":True,"dead_letter":2}], "links": []}
|
||||
await nm.check_and_alert(status=s)
|
||||
assert any("dead-letter" in t for t in sent)
|
||||
|
||||
Reference in New Issue
Block a user