From cdc309150e3b3901a9d83d20699caa66a85369e6 Mon Sep 17 00:00:00 2001 From: gahusb Date: Thu, 9 Jul 2026 17:09:59 +0900 Subject: [PATCH] =?UTF-8?q?feat(realestate):=20=EB=A7=A4=EB=AC=BC=20?= =?UTF-8?q?=EC=88=98=EC=A7=91=20MOLIT/Naver=20=EC=A7=84=EB=8B=A8=20?= =?UTF-8?q?=EA=B3=84=EC=B8=A1(collect=5Flog.error)=20=E2=80=94=20silent-fa?= =?UTF-8?q?ilure=20=EA=B4=80=EC=B8=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EqCYBhvTcdeCTUDX3RhWx9 --- realestate-lab/app/listing_collector.py | 53 +++++++++++++------ .../tests/test_listing_collector.py | 30 ++++++++++- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/realestate-lab/app/listing_collector.py b/realestate-lab/app/listing_collector.py index 5778599..e4e66bd 100644 --- a/realestate-lab/app/listing_collector.py +++ b/realestate-lab/app/listing_collector.py @@ -75,11 +75,13 @@ def _parse_naver_article(raw: dict) -> dict: "raw_json": None} -def _molit_call(op: str, lawd: str, ym: str) -> list: - """국토부 실거래 단건 호출 (기존 collector 재시도 관용). 실패 시 [].""" +def _molit_call(op: str, lawd: str, ym: str): + """국토부 실거래 단건 호출 (재시도). Returns (items, diag). + diag=None on success, else 짧은 진단 문자열(http=401 / json_err / req_err=...) — 관측용.""" if not MOLIT_KEY: - return [] + return [], "no_key" url = f"https://apis.data.go.kr/1613000/RTMSDataSvc{op[len('getRTMSDataSvc'):]}/{op}" + last_diag = None for attempt in range(3): try: resp = requests.get(url, params={"serviceKey": MOLIT_KEY, "LAWD_CD": lawd, @@ -87,18 +89,28 @@ def _molit_call(op: str, lawd: str, ym: str) -> list: resp.raise_for_status() body = resp.json() items = (((body.get("response") or {}).get("body") or {}).get("items") or {}).get("item") or [] - return items if isinstance(items, list) else [items] + return (items if isinstance(items, list) else [items]), None + except requests.HTTPError as e: + code = getattr(getattr(e, "response", None), "status_code", "?") + last_diag = f"http={code}" except requests.RequestException as e: - if attempt < 2: - time.sleep(2 ** attempt) - else: - logger.error("MOLIT %s 실패: %s", op, e) - return [] + last_diag = f"req_err={type(e).__name__}" + except ValueError: + # 200이지만 JSON 아님(XML 오류 응답 등) — 재시도 무의미 + last_diag = "json_err" + break + if attempt < 2: + time.sleep(2 ** attempt) + logger.error("MOLIT %s 실패: %s", op, last_diag) + return [], last_diag -def _fetch_molit_all() -> int: - """대상 동 시군구코드 × 유형 × 최근 2개월 전월세·매매 실거래 수집→market_deals. 저장건수 반환.""" +def _fetch_molit_all(): + """대상 동 시군구코드 × 유형 × 최근 2개월 전월세·매매 실거래 수집→market_deals. + Returns (saved_count, diag). diag = 'calls=N saved=M [err=<첫에러>]' 관측용.""" saved = 0 + calls = 0 + first_err = None lawds = set(LAWD_CODES.values()) today = date.today() yms = [today.strftime("%Y%m"), @@ -107,7 +119,11 @@ def _fetch_molit_all() -> int: for htype, (rent_op, trade_op) in MOLIT_OPS.items(): for ym in yms: for op, parser in ((rent_op, _parse_molit_rent), (trade_op, _parse_molit_trade)): - for raw in _molit_call(op, lawd, ym): + items, diag = _molit_call(op, lawd, ym) + calls += 1 + if diag and first_err is None: + first_err = diag + for raw in items: try: d = parser(raw, htype) d["dong_code"] = lawd @@ -117,7 +133,8 @@ def _fetch_molit_all() -> int: except Exception as e: logger.warning("MOLIT 파싱 실패: %s", e) time.sleep(0.3) - return saved + diag = f"calls={calls} saved={saved}" + (f" err={first_err}" if first_err else "") + return saved, diag def _fetch_naver_all() -> tuple: @@ -153,14 +170,18 @@ def _fetch_naver_all() -> tuple: def collect_listings() -> dict: naver_ok = True new_count = total = 0 + molit_diag = naver_diag = None try: - _fetch_molit_all() # 실거래(합법)는 항상 우선 — 안전마진 baseline + _, molit_diag = _fetch_molit_all() # 실거래(합법)는 항상 우선 — 안전마진 baseline except Exception as e: + molit_diag = f"exc={type(e).__name__}:{str(e)[:80]}" logger.error("MOLIT 수집 오류(안전마진 baseline): %s", e) try: new_count, total = _fetch_naver_all() except requests.RequestException as e: naver_ok = False + naver_diag = f"{type(e).__name__}:{str(e)[:80]}" logger.warning("네이버 매물수집 skip(차단/오류) — 안전마진·실거래는 유지: %s", e) - save_listing_collect_log(new_count, total, naver_ok) - return {"new_count": new_count, "total_count": total, "naver_ok": naver_ok} + diag = f"molit[{molit_diag}] naver[{naver_diag or 'ok'}]" + save_listing_collect_log(new_count, total, naver_ok, error=diag) + return {"new_count": new_count, "total_count": total, "naver_ok": naver_ok, "diag": diag} diff --git a/realestate-lab/tests/test_listing_collector.py b/realestate-lab/tests/test_listing_collector.py index c0bb3dc..fde55f3 100644 --- a/realestate-lab/tests/test_listing_collector.py +++ b/realestate-lab/tests/test_listing_collector.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import patch, MagicMock def test_parse_molit_rent_fields(): from app.listing_collector import _parse_molit_rent @@ -20,10 +20,36 @@ def test_parse_naver_article_fields(): def test_collect_naver_block_sets_naver_ok_false(monkeypatch): """네이버 403 → 매물수집 skip + naver_ok=False, 실거래는 계속.""" from app import listing_collector as lc - monkeypatch.setattr(lc, "_fetch_molit_all", lambda: 2) # 실거래 2건 저장됨 가정 + monkeypatch.setattr(lc, "_fetch_molit_all", lambda: (2, "calls=1 saved=2")) def boom(*a, **k): import requests raise requests.RequestException("403") monkeypatch.setattr(lc, "_fetch_naver_all", boom) r = lc.collect_listings() assert r["naver_ok"] is False # 안전마진(실거래)은 유지 + +def test_molit_call_returns_diag_on_http_error(monkeypatch): + """MOLIT 401 등 HTTP 오류 → ([], diag) 반환, diag에 상태코드 포함(관측).""" + from app import listing_collector as lc + import requests + monkeypatch.setattr(lc, "MOLIT_KEY", "TESTKEY") + monkeypatch.setattr(lc.time, "sleep", lambda *a: None) # 재시도 대기 skip + resp = MagicMock() + resp.status_code = 401 + resp.raise_for_status.side_effect = requests.HTTPError(response=resp) + monkeypatch.setattr(lc.requests, "get", lambda *a, **k: resp) + items, diag = lc._molit_call("getRTMSDataSvcAptTradeDev", "11590", "202606") + assert items == [] + assert diag and "401" in diag + +def test_collect_writes_molit_diag_to_log(monkeypatch): + """collect_listings가 MOLIT/Naver 진단을 collect_log.error에 기록(silent-failure 관측).""" + from app import listing_collector as lc + from app import db + monkeypatch.setattr(lc, "_fetch_molit_all", lambda: (0, "calls=36 saved=0 err=http=401")) + monkeypatch.setattr(lc, "_fetch_naver_all", lambda: (0, 0)) + lc.collect_listings() + log = db.get_last_listing_collect_log() + assert log is not None + assert "http=401" in (log["error"] or "") + assert "molit" in (log["error"] or "")