feat(agent-office): 노드 헬스 1분 cron + 텔레그램 경보(다운/복구/dead-letter)

agent-office/app/node_monitor.py

@@ -2,10 +2,13 @@
 from __future__ import annotations
 import datetime as dt, json, logging
 import redis.asyncio as aioredis
-from .config import REDIS_URL
+from .config import REDIS_URL, NODE_ALERT_DEADLETTER_THRESHOLD
 
 logger = logging.getLogger("agent-office.node_monitor")
 
+_node_state: dict[str, bool] = {}   # name -> 직전 alive
+_dl_notified: dict[str, int] = {}   # name -> 직전 알린 dead_letter 수
+
 WORKER_REGISTRY = [
     {"name": "music-render", "kind": "render", "queue": "queue:music-render"},
     {"name": "video-render", "kind": "render", "queue": "queue:video-render"},
@@ -94,3 +97,39 @@ async def collect_status(redis=None) -> dict:
     if out["paused"] and not out["paused_reason"]:
         out["paused_reason"] = "trading"
     return out
+
+
+async def check_and_alert(status=None) -> list[str]:
+    """워커 상태를 점검해 다운/복구/dead-letter 전이를 텔레그램으로 경보한다.
+
+    첫 관측(prev=None)엔 경보 없음 — 부팅 시 false alarm 방지.
+    반환값: 실제로 전송된 경보 텍스트 목록 (테스트용).
+    """
+    from .telegram.messaging import send_raw
+    from .db import add_log
+    st = status or await collect_status()
+    sent: list[str] = []
+    for w in st["workers"]:
+        name, alive = w["name"], w.get("alive", False)
+        prev = _node_state.get(name)
+        if prev is True and not alive:
+            text = f"🔴 [{name}] 워커 다운"
+            if (await send_raw(text=text)).get("ok"):
+                add_log("node_monitor", f"{name} 다운", "warning")
+                sent.append(text)
+        elif prev is False and alive:
+            text = f"🟢 [{name}] 워커 복구"
+            if (await send_raw(text=text)).get("ok"):
+                add_log("node_monitor", f"{name} 복구", "info")
+                sent.append(text)
+        _node_state[name] = alive
+        dl = w.get("dead_letter", 0)
+        if dl >= NODE_ALERT_DEADLETTER_THRESHOLD and dl != _dl_notified.get(name, 0):
+            text = f"❌ [{name}] 실패 누적 {dl}건 (dead-letter)"
+            if (await send_raw(text=text)).get("ok"):
+                add_log("node_monitor", f"{name} dead-letter {dl}", "warning")
+                sent.append(text)
+            _dl_notified[name] = dl
+        elif dl == 0:
+            _dl_notified.pop(name, None)
+    return sent