feat(render-workers): 4 render 워커 heartbeat 배선 + poll_once 카운터
- services/_shared/heartbeat.py (A1) WorkerStats/utc_now_iso/heartbeat_loop 소비 - image-render / video-render / music-render / insta-render 각 worker.py: stats = WorkerStats() 모듈 레벨 추가, poll_once에서 dispatch 전 busy=True, ack 후 jobs_done+1 / fail 후 jobs_failed+1 + last_job_at + busy=False - 각 main.py: lifespan에 aioredis(decode_responses=False) + heartbeat_loop 태스크 spawn, 종료 시 cancel + aclose - 각 tests/test_worker.py: test_poll_once_increments_jobs_done 추가 (image:flux / video:sora / music:suno / insta:_process_one mock) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019LV86jBozkNhSFXJA412fq
This commit is contained in:
@@ -3,12 +3,15 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from fastapi import FastAPI
|
||||
|
||||
import card_renderer
|
||||
import worker
|
||||
from _shared.heartbeat import heartbeat_loop
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -20,15 +23,19 @@ async def lifespan(app: FastAPI):
|
||||
await card_renderer.init_browser()
|
||||
# 큐 워커 백그라운드 시작
|
||||
worker_task = asyncio.create_task(worker.worker_loop())
|
||||
hb_redis = aioredis.from_url(os.getenv("REDIS_URL", "redis://192.168.45.54:6379"), decode_responses=False)
|
||||
hb_task = asyncio.create_task(heartbeat_loop(hb_redis, "insta-render", "render", worker.stats))
|
||||
logger.info("insta-render lifespan 시작")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
worker_task.cancel()
|
||||
try:
|
||||
await worker_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
for t in (worker_task, hb_task):
|
||||
t.cancel()
|
||||
try:
|
||||
await t
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await hb_redis.aclose()
|
||||
await card_renderer.shutdown_browser()
|
||||
logger.info("insta-render lifespan 종료")
|
||||
|
||||
|
||||
@@ -230,3 +230,27 @@ def test_make_queue_redis_socket_timeout_exceeds_block():
|
||||
c = worker.make_queue_redis()
|
||||
st = c.connection_pool.connection_kwargs.get("socket_timeout")
|
||||
assert st is not None and st > 5 # blmove 블록(5s)보다 커야 안정
|
||||
|
||||
|
||||
# ----- heartbeat stats 카운터 -----
|
||||
|
||||
class _OneJobQueueInsta:
|
||||
def __init__(self): self.acked = False
|
||||
async def dequeue(self, timeout=5):
|
||||
if self.acked: return None
|
||||
return ({"task_id": "t1", "params": {"slate_id": 1, "theme": "default"}}, b"raw")
|
||||
async def ack(self, raw): self.acked = True
|
||||
async def fail(self, raw, payload): pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_poll_once_increments_jobs_done(monkeypatch):
|
||||
worker.stats.jobs_done = 0
|
||||
async def fake_process(client, payload): pass
|
||||
monkeypatch.setattr(worker, "_process_one", fake_process)
|
||||
async with httpx.AsyncClient() as client:
|
||||
handled = await worker.poll_once(_OneJobQueueInsta(), client)
|
||||
assert handled is True
|
||||
assert worker.stats.jobs_done == 1
|
||||
assert worker.stats.busy is False
|
||||
assert worker.stats.last_job_at is not None
|
||||
|
||||
@@ -14,9 +14,11 @@ import redis.asyncio as aioredis
|
||||
|
||||
from card_renderer import render_slate
|
||||
from _shared.reliable_queue import ReliableQueue
|
||||
from _shared.heartbeat import WorkerStats, utc_now_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
stats = WorkerStats()
|
||||
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://192.168.45.54:6379")
|
||||
NAS_BASE_URL = os.getenv("NAS_BASE_URL", "http://192.168.45.54:18700")
|
||||
@@ -89,12 +91,19 @@ async def poll_once(queue: ReliableQueue, client: httpx.AsyncClient) -> bool:
|
||||
if result is None:
|
||||
return False
|
||||
payload, raw = result
|
||||
stats.busy = True
|
||||
try:
|
||||
await _process_one(client, payload)
|
||||
except Exception:
|
||||
await queue.fail(raw, payload)
|
||||
stats.jobs_failed += 1
|
||||
stats.last_job_at = utc_now_iso()
|
||||
stats.busy = False
|
||||
return True
|
||||
await queue.ack(raw)
|
||||
stats.jobs_done += 1
|
||||
stats.last_job_at = utc_now_iso()
|
||||
stats.busy = False
|
||||
return True
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user