feat(agent-office): 노드 헬스 1분 cron + 텔레그램 경보(다운/복구/dead-letter)

This commit is contained in:
2026-06-29 18:06:38 +09:00
parent 2a0090a1d4
commit 5d5ff27d29
3 changed files with 73 additions and 1 deletions

View File

@@ -96,3 +96,31 @@ async def test_llen_exception_returns_redis_ok_false():
)
st = await node_monitor.collect_status(redis=r)
assert st["redis_ok"] is False
import app.node_monitor as nm
@pytest.mark.asyncio
async def test_alert_on_alive_to_dead(monkeypatch):
sent = []
async def fake_send_raw(text, **kw): sent.append(text); return {"ok": True}
monkeypatch.setattr("app.telegram.messaging.send_raw", fake_send_raw)
monkeypatch.setattr("app.db.add_log", lambda *a, **k: None)
nm._node_state.clear(); nm._dl_notified.clear()
alive = {"workers": [{"name":"image-render","alive":True,"dead_letter":0}], "links": []}
dead = {"workers": [{"name":"image-render","alive":False,"dead_letter":0}], "links": []}
await nm.check_and_alert(status=alive) # 첫 관측 — 경보 없음
assert sent == []
await nm.check_and_alert(status=dead) # alive→dead 전이
assert any("다운" in t for t in sent)
@pytest.mark.asyncio
async def test_alert_on_dead_letter_growth(monkeypatch):
sent = []
async def fake_send_raw(text, **kw): sent.append(text); return {"ok": True}
monkeypatch.setattr("app.telegram.messaging.send_raw", fake_send_raw)
monkeypatch.setattr("app.db.add_log", lambda *a, **k: None)
nm._node_state.clear(); nm._dl_notified.clear()
s = {"workers": [{"name":"video-render","alive":True,"dead_letter":2}], "links": []}
await nm.check_and_alert(status=s)
assert any("dead-letter" in t for t in sent)