feat(realestate): 매물 수집 MOLIT/Naver 진단 계측(collect_log.error) — silent-failure 관측

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EqCYBhvTcdeCTUDX3RhWx9

realestate-lab/app/listing_collector.py

@@ -75,11 +75,13 @@ def _parse_naver_article(raw: dict) -> dict:
             "raw_json": None}
 
 
-def _molit_call(op: str, lawd: str, ym: str) -> list:
+def _molit_call(op: str, lawd: str, ym: str):
-    """국토부 실거래 단건 호출 (기존 collector 재시도 관용). 실패 시 []."""
+    """국토부 실거래 단건 호출 (재시도). Returns (items, diag).
+    diag=None on success, else 짧은 진단 문자열(http=401 / json_err / req_err=...) — 관측용."""
     if not MOLIT_KEY:
-        return []
+        return [], "no_key"
     url = f"https://apis.data.go.kr/1613000/RTMSDataSvc{op[len('getRTMSDataSvc'):]}/{op}"
+    last_diag = None
     for attempt in range(3):
         try:
             resp = requests.get(url, params={"serviceKey": MOLIT_KEY, "LAWD_CD": lawd,
@@ -87,18 +89,28 @@ def _molit_call(op: str, lawd: str, ym: str) -> list:
             resp.raise_for_status()
             body = resp.json()
             items = (((body.get("response") or {}).get("body") or {}).get("items") or {}).get("item") or []
-            return items if isinstance(items, list) else [items]
+            return (items if isinstance(items, list) else [items]), None
+        except requests.HTTPError as e:
+            code = getattr(getattr(e, "response", None), "status_code", "?")
+            last_diag = f"http={code}"
         except requests.RequestException as e:
+            last_diag = f"req_err={type(e).__name__}"
+        except ValueError:
+            # 200이지만 JSON 아님(XML 오류 응답 등) — 재시도 무의미
+            last_diag = "json_err"
+            break
         if attempt < 2:
             time.sleep(2 ** attempt)
-            else:
+    logger.error("MOLIT %s 실패: %s", op, last_diag)
-                logger.error("MOLIT %s 실패: %s", op, e)
+    return [], last_diag
-    return []
 
 
-def _fetch_molit_all() -> int:
+def _fetch_molit_all():
-    """대상 동 시군구코드 × 유형 × 최근 2개월 전월세·매매 실거래 수집→market_deals. 저장건수 반환."""
+    """대상 동 시군구코드 × 유형 × 최근 2개월 전월세·매매 실거래 수집→market_deals.
+    Returns (saved_count, diag). diag = 'calls=N saved=M [err=<첫에러>]' 관측용."""
     saved = 0
+    calls = 0
+    first_err = None
     lawds = set(LAWD_CODES.values())
     today = date.today()
     yms = [today.strftime("%Y%m"),
@@ -107,7 +119,11 @@ def _fetch_molit_all() -> int:
         for htype, (rent_op, trade_op) in MOLIT_OPS.items():
             for ym in yms:
                 for op, parser in ((rent_op, _parse_molit_rent), (trade_op, _parse_molit_trade)):
-                    for raw in _molit_call(op, lawd, ym):
+                    items, diag = _molit_call(op, lawd, ym)
+                    calls += 1
+                    if diag and first_err is None:
+                        first_err = diag
+                    for raw in items:
                         try:
                             d = parser(raw, htype)
                             d["dong_code"] = lawd
@@ -117,7 +133,8 @@ def _fetch_molit_all() -> int:
                         except Exception as e:
                             logger.warning("MOLIT 파싱 실패: %s", e)
                     time.sleep(0.3)
-    return saved
+    diag = f"calls={calls} saved={saved}" + (f" err={first_err}" if first_err else "")
+    return saved, diag
 
 
 def _fetch_naver_all() -> tuple:
@@ -153,14 +170,18 @@ def _fetch_naver_all() -> tuple:
 def collect_listings() -> dict:
     naver_ok = True
     new_count = total = 0
+    molit_diag = naver_diag = None
     try:
-        _fetch_molit_all()   # 실거래(합법)는 항상 우선 — 안전마진 baseline
+        _, molit_diag = _fetch_molit_all()   # 실거래(합법)는 항상 우선 — 안전마진 baseline
     except Exception as e:
+        molit_diag = f"exc={type(e).__name__}:{str(e)[:80]}"
         logger.error("MOLIT 수집 오류(안전마진 baseline): %s", e)
     try:
         new_count, total = _fetch_naver_all()
     except requests.RequestException as e:
         naver_ok = False
+        naver_diag = f"{type(e).__name__}:{str(e)[:80]}"
         logger.warning("네이버 매물수집 skip(차단/오류) — 안전마진·실거래는 유지: %s", e)
-    save_listing_collect_log(new_count, total, naver_ok)
+    diag = f"molit[{molit_diag}] naver[{naver_diag or 'ok'}]"
-    return {"new_count": new_count, "total_count": total, "naver_ok": naver_ok}
+    save_listing_collect_log(new_count, total, naver_ok, error=diag)
+    return {"new_count": new_count, "total_count": total, "naver_ok": naver_ok, "diag": diag}

realestate-lab/tests/test_listing_collector.py

@@ -1,4 +1,4 @@
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock
 
 def test_parse_molit_rent_fields():
     from app.listing_collector import _parse_molit_rent
@@ -20,10 +20,36 @@ def test_parse_naver_article_fields():
 def test_collect_naver_block_sets_naver_ok_false(monkeypatch):
     """네이버 403 → 매물수집 skip + naver_ok=False, 실거래는 계속."""
     from app import listing_collector as lc
-    monkeypatch.setattr(lc, "_fetch_molit_all", lambda: 2)      # 실거래 2건 저장됨 가정
+    monkeypatch.setattr(lc, "_fetch_molit_all", lambda: (2, "calls=1 saved=2"))
     def boom(*a, **k):
         import requests
         raise requests.RequestException("403")
     monkeypatch.setattr(lc, "_fetch_naver_all", boom)
     r = lc.collect_listings()
     assert r["naver_ok"] is False   # 안전마진(실거래)은 유지
+
+def test_molit_call_returns_diag_on_http_error(monkeypatch):
+    """MOLIT 401 등 HTTP 오류 → ([], diag) 반환, diag에 상태코드 포함(관측)."""
+    from app import listing_collector as lc
+    import requests
+    monkeypatch.setattr(lc, "MOLIT_KEY", "TESTKEY")
+    monkeypatch.setattr(lc.time, "sleep", lambda *a: None)  # 재시도 대기 skip
+    resp = MagicMock()
+    resp.status_code = 401
+    resp.raise_for_status.side_effect = requests.HTTPError(response=resp)
+    monkeypatch.setattr(lc.requests, "get", lambda *a, **k: resp)
+    items, diag = lc._molit_call("getRTMSDataSvcAptTradeDev", "11590", "202606")
+    assert items == []
+    assert diag and "401" in diag
+
+def test_collect_writes_molit_diag_to_log(monkeypatch):
+    """collect_listings가 MOLIT/Naver 진단을 collect_log.error에 기록(silent-failure 관측)."""
+    from app import listing_collector as lc
+    from app import db
+    monkeypatch.setattr(lc, "_fetch_molit_all", lambda: (0, "calls=36 saved=0 err=http=401"))
+    monkeypatch.setattr(lc, "_fetch_naver_all", lambda: (0, 0))
+    lc.collect_listings()
+    log = db.get_last_listing_collect_log()
+    assert log is not None
+    assert "http=401" in (log["error"] or "")
+    assert "molit" in (log["error"] or "")