- services/_shared/heartbeat.py (A1) WorkerStats/utc_now_iso/heartbeat_loop 소비 - image-render / video-render / music-render / insta-render 각 worker.py: stats = WorkerStats() 모듈 레벨 추가, poll_once에서 dispatch 전 busy=True, ack 후 jobs_done+1 / fail 후 jobs_failed+1 + last_job_at + busy=False - 각 main.py: lifespan에 aioredis(decode_responses=False) + heartbeat_loop 태스크 spawn, 종료 시 cancel + aclose - 각 tests/test_worker.py: test_poll_once_increments_jobs_done 추가 (image:flux / video:sora / music:suno / insta:_process_one mock) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019LV86jBozkNhSFXJA412fq
192 lines
6.9 KiB
Python
192 lines
6.9 KiB
Python
"""worker.py — job_type 디스패처 + paused 체크."""
|
|
import json
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import worker
|
|
|
|
|
|
def test_dispatch_suno_generation_calls_run_suno_generation():
|
|
payload = {
|
|
"task_id": "t1",
|
|
"job_type": "suno_generation",
|
|
"params": {"genre": "lofi", "title": "x"},
|
|
}
|
|
with patch("worker.run_suno_generation") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t1", {"genre": "lofi", "title": "x"})
|
|
|
|
|
|
def test_dispatch_local_generation_calls_run_local_generation():
|
|
payload = {
|
|
"task_id": "t2",
|
|
"job_type": "local_generation",
|
|
"params": {"genre": "ambient"},
|
|
}
|
|
with patch("worker.run_local_generation") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t2", {"genre": "ambient"})
|
|
|
|
|
|
def test_dispatch_unknown_job_type_logs_error():
|
|
payload = {"task_id": "t3", "job_type": "weird_type", "params": {}}
|
|
with patch("worker.webhook_update_task") as m:
|
|
worker._dispatch(payload)
|
|
# 알 수 없는 job_type은 failed로 보고
|
|
m.assert_called_once()
|
|
args = m.call_args[0]
|
|
assert args[0] == "t3"
|
|
assert args[1] == "failed"
|
|
|
|
|
|
def test_dispatch_suno_extend_calls_run_suno_extend():
|
|
payload = {"task_id": "t4", "job_type": "suno_extend", "params": {"suno_id": "abc"}}
|
|
with patch("worker.run_suno_extend") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t4", {"suno_id": "abc"})
|
|
|
|
|
|
def test_dispatch_vocal_removal_calls_run_vocal_removal():
|
|
payload = {"task_id": "t5", "job_type": "vocal_removal", "params": {"suno_id": "abc"}}
|
|
with patch("worker.run_vocal_removal") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t5", {"suno_id": "abc"})
|
|
|
|
|
|
def test_dispatch_cover_image_calls_run_cover_image():
|
|
payload = {"task_id": "t6", "job_type": "cover_image", "params": {"suno_task_id": "x"}}
|
|
with patch("worker.run_cover_image") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t6", {"suno_task_id": "x"})
|
|
|
|
|
|
def test_dispatch_wav_convert_calls_run_wav_convert():
|
|
payload = {"task_id": "t7", "job_type": "wav_convert", "params": {"suno_task_id": "x", "suno_id": "y"}}
|
|
with patch("worker.run_wav_convert") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t7", {"suno_task_id": "x", "suno_id": "y"})
|
|
|
|
|
|
def test_dispatch_stem_split_calls_run_stem_split():
|
|
payload = {"task_id": "t8", "job_type": "stem_split", "params": {"suno_task_id": "x", "suno_id": "y"}}
|
|
with patch("worker.run_stem_split") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t8", {"suno_task_id": "x", "suno_id": "y"})
|
|
|
|
|
|
def test_dispatch_video_generate_calls_run_video_generate():
|
|
payload = {"task_id": "t9", "job_type": "video_generate", "params": {"suno_task_id": "x", "suno_id": "y"}}
|
|
with patch("worker.run_video_generate") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t9", {"suno_task_id": "x", "suno_id": "y"})
|
|
|
|
|
|
def test_dispatch_upload_cover_calls_run_upload_cover():
|
|
payload = {"task_id": "t10", "job_type": "upload_cover", "params": {"upload_url": "u"}}
|
|
with patch("worker.run_upload_cover") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t10", {"upload_url": "u"})
|
|
|
|
|
|
def test_dispatch_upload_extend_calls_run_upload_extend():
|
|
payload = {"task_id": "t11", "job_type": "upload_extend", "params": {"upload_url": "u"}}
|
|
with patch("worker.run_upload_extend") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t11", {"upload_url": "u"})
|
|
|
|
|
|
def test_dispatch_add_vocals_calls_run_add_vocals():
|
|
payload = {"task_id": "t12", "job_type": "add_vocals", "params": {"upload_url": "u"}}
|
|
with patch("worker.run_add_vocals") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t12", {"upload_url": "u"})
|
|
|
|
|
|
def test_dispatch_add_instrumental_calls_run_add_instrumental():
|
|
payload = {"task_id": "t13", "job_type": "add_instrumental", "params": {"upload_url": "u"}}
|
|
with patch("worker.run_add_instrumental") as m:
|
|
worker._dispatch(payload)
|
|
m.assert_called_once_with("t13", {"upload_url": "u"})
|
|
|
|
|
|
# ----- F6: ReliableQueue poll_once -----
|
|
|
|
from unittest.mock import AsyncMock
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_poll_once_acks_on_success(monkeypatch):
|
|
"""F6 — _dispatch 정상 return → queue.ack(raw)."""
|
|
payload = {"task_id": "t1", "job_type": "suno_generation", "params": {}}
|
|
raw = json.dumps(payload).encode()
|
|
fake_queue = AsyncMock()
|
|
fake_queue.dequeue = AsyncMock(return_value=(payload, raw))
|
|
fake_queue.ack = AsyncMock()
|
|
fake_queue.fail = AsyncMock()
|
|
|
|
monkeypatch.setattr(worker, "_dispatch", MagicMock())
|
|
|
|
handled = await worker.poll_once(fake_queue)
|
|
assert handled is True
|
|
fake_queue.ack.assert_awaited_once_with(raw)
|
|
fake_queue.fail.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_poll_once_calls_fail_on_dispatch_exception(monkeypatch):
|
|
"""F6 — _dispatch unhandled exception → queue.fail(raw, payload)."""
|
|
payload = {"task_id": "t2", "job_type": "suno_generation", "params": {}}
|
|
raw = json.dumps(payload).encode()
|
|
fake_queue = AsyncMock()
|
|
fake_queue.dequeue = AsyncMock(return_value=(payload, raw))
|
|
fake_queue.ack = AsyncMock()
|
|
fake_queue.fail = AsyncMock()
|
|
|
|
def _boom(p):
|
|
raise RuntimeError("dispatch crash")
|
|
|
|
monkeypatch.setattr(worker, "_dispatch", _boom)
|
|
|
|
handled = await worker.poll_once(fake_queue)
|
|
assert handled is True
|
|
fake_queue.fail.assert_awaited_once_with(raw, payload)
|
|
fake_queue.ack.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_poll_once_returns_false_on_timeout(monkeypatch):
|
|
fake_queue = AsyncMock()
|
|
fake_queue.dequeue = AsyncMock(return_value=None)
|
|
fake_queue.ack = AsyncMock()
|
|
fake_queue.fail = AsyncMock()
|
|
dispatch_mock = MagicMock()
|
|
monkeypatch.setattr(worker, "_dispatch", dispatch_mock)
|
|
|
|
handled = await worker.poll_once(fake_queue)
|
|
assert handled is False
|
|
dispatch_mock.assert_not_called()
|
|
fake_queue.ack.assert_not_awaited()
|
|
fake_queue.fail.assert_not_awaited()
|
|
|
|
|
|
# ----- heartbeat stats 카운터 -----
|
|
|
|
class _OneJobQueue:
|
|
def __init__(self): self.acked = False
|
|
async def dequeue(self, timeout=5):
|
|
if self.acked: return None
|
|
return ({"job_type": "suno_generation", "task_id": "t1", "params": {}}, b"raw")
|
|
async def ack(self, raw): self.acked = True
|
|
async def fail(self, raw, payload): pass
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_poll_once_increments_jobs_done(monkeypatch):
|
|
worker.stats.jobs_done = 0
|
|
monkeypatch.setattr(worker, "run_suno_generation", lambda task_id, params: None)
|
|
handled = await worker.poll_once(_OneJobQueue())
|
|
assert handled is True
|
|
assert worker.stats.jobs_done == 1
|
|
assert worker.stats.busy is False
|
|
assert worker.stats.last_job_at is not None
|