web-page-backend/realestate-lab/app/listing_collector.py

"""매물 수집: 국토부 실거래가(합법, market_deals) + 네이버부동산 호가(회색·폴백, listings)."""
import os
import re
import time
import logging
import requests
from datetime import date

from .lawd_codes import LAWD_CODES
from .db import (get_listing_criteria, upsert_listing, upsert_market_deal,
                 save_listing_collect_log)

logger = logging.getLogger("realestate-lab")

MOLIT_KEY = os.getenv("DATA_GO_KR_API_KEY", "")
NAVER_BASE = "https://new.land.naver.com/api"
# MOLIT operation ID (스펙 §3, 라이브 확인 필요). house_type → (rent_op, trade_op)
MOLIT_OPS = {
    "아파트": ("getRTMSDataSvcAptRent", "getRTMSDataSvcAptTradeDev"),
    "오피스텔": ("getRTMSDataSvcOffiRent", "getRTMSDataSvcOffiTrade"),
    "연립다세대": ("getRTMSDataSvcRHRent", "getRTMSDataSvcRHTrade"),
}
_HOUSE_TYPE_ALIAS = {"빌라": "연립다세대"}


def _won_to_manwon(s) -> int | None:
    """'43,000' 또는 '2억 9,000' → 만원 정수."""
    if s is None:
        return None
    t = str(s).replace(",", "").strip()
    if not t:
        return None
    m = re.match(r"(?:(\d+)억)?\s*(\d+)?", t)
    if m and (m.group(1) or m.group(2)):
        eok = int(m.group(1)) if m.group(1) else 0
        man = int(m.group(2)) if m.group(2) else 0
        return eok * 10000 + man
    try:
        return int(float(t))
    except ValueError:
        return None


def _parse_molit_rent(raw: dict, house_type: str) -> dict:
    deposit = _won_to_manwon(raw.get("deposit"))
    rent = _won_to_manwon(raw.get("monthlyRent"))
    return {"house_type": house_type, "complex_name": (raw.get("aptNm") or raw.get("offiNm") or "").strip(),
            "area": float(raw.get("excluUseAr") or 0) or None, "deal_type": "전세" if not rent else "월세",
            "deposit": deposit, "monthly_rent": rent, "sale_amount": None,
            "deal_ym": f"{raw.get('dealYear','')}{str(raw.get('dealMonth','')).zfill(2)}",
            "floor": str(raw.get("floor") or ""), "source": "molit"}


def _parse_molit_trade(raw: dict, house_type: str) -> dict:
    return {"house_type": house_type, "complex_name": (raw.get("aptNm") or raw.get("offiNm") or "").strip(),
            "area": float(raw.get("excluUseAr") or 0) or None, "deal_type": "매매",
            "deposit": None, "monthly_rent": None,
            "sale_amount": _won_to_manwon(raw.get("dealAmount")),
            "deal_ym": f"{raw.get('dealYear','')}{str(raw.get('dealMonth','')).zfill(2)}",
            "floor": str(raw.get("floor") or ""), "source": "molit"}


def _parse_naver_article(raw: dict) -> dict:
    dt = raw.get("tradeTypeName") or ""
    prc = _won_to_manwon(raw.get("dealOrWarrantPrc"))
    rent = _won_to_manwon(raw.get("rentPrc"))
    is_sale = dt == "매매"
    return {"article_no": str(raw.get("articleNo") or ""), "source": "naver", "deal_type": dt,
            "deposit": None if is_sale else prc, "monthly_rent": rent,
            "sale_price": prc if is_sale else None,
            "area_exclusive": float(raw.get("area2") or raw.get("areaName") or 0) or None,
            "floor": raw.get("floorInfo"), "complex_name": raw.get("articleName") or raw.get("buildingName"),
            "dong_code": (raw.get("cortarNo") or "")[:5] or None,
            "url": f"https://new.land.naver.com/houses?articleNo={raw.get('articleNo')}",
            "raw_json": None}


def _molit_call(op: str, lawd: str, ym: str) -> list:
    """국토부 실거래 단건 호출 (기존 collector 재시도 관용). 실패 시 []."""
    if not MOLIT_KEY:
        return []
    url = f"https://apis.data.go.kr/1613000/RTMSDataSvc{op[len('getRTMSDataSvc'):]}/{op}"
    for attempt in range(3):
        try:
            resp = requests.get(url, params={"serviceKey": MOLIT_KEY, "LAWD_CD": lawd,
                                             "DEAL_YMD": ym, "numOfRows": 200, "_type": "json"}, timeout=30)
            resp.raise_for_status()
            body = resp.json()
            items = (((body.get("response") or {}).get("body") or {}).get("items") or {}).get("item") or []
            return items if isinstance(items, list) else [items]
        except requests.RequestException as e:
            if attempt < 2:
                time.sleep(2 ** attempt)
            else:
                logger.error("MOLIT %s 실패: %s", op, e)
    return []


def _fetch_molit_all() -> int:
    """대상 동 시군구코드 × 유형 × 최근 2개월 전월세·매매 실거래 수집→market_deals. 저장건수 반환."""
    saved = 0
    lawds = set(LAWD_CODES.values())
    today = date.today()
    yms = [today.strftime("%Y%m"),
           (today.replace(day=1) - __import__("datetime").timedelta(days=1)).strftime("%Y%m")]
    for lawd in lawds:
        for htype, (rent_op, trade_op) in MOLIT_OPS.items():
            for ym in yms:
                for op, parser in ((rent_op, _parse_molit_rent), (trade_op, _parse_molit_trade)):
                    for raw in _molit_call(op, lawd, ym):
                        try:
                            d = parser(raw, htype)
                            d["dong_code"] = lawd
                            d["dong"] = (raw.get("umdNm") or "").strip() or None
                            upsert_market_deal(d)
                            saved += 1
                        except Exception as e:
                            logger.warning("MOLIT 파싱 실패: %s", e)
                    time.sleep(0.3)
    return saved


def _fetch_naver_all() -> tuple:
    """대상 동 네이버 호가 매물 수집→listings. (new_count, total_count) 반환. 예외는 상위로 전파."""
    new_count = total = 0
    crit = get_listing_criteria()
    for dong in crit.get("dongs", []):
        code = LAWD_CODES.get(dong)
        if not code:
            continue
        # 저빈도: 동당 1회, UA/Referer 현실값. 응답 articleList 순회.
        resp = requests.get(f"{NAVER_BASE}/articles",
                            params={"cortarNo": code, "order": "dateDesc", "page": 1},
                            headers={"User-Agent": "Mozilla/5.0", "Referer": "https://new.land.naver.com/"},
                            timeout=20)
        resp.raise_for_status()
        for raw in (resp.json().get("articleList") or []):
            try:
                d = _parse_naver_article(raw)
                if not d["article_no"]:
                    continue
                d["dong"] = dong
                _, is_new = upsert_listing(d)
                total += 1
                if is_new:
                    new_count += 1
            except Exception as e:
                logger.warning("네이버 파싱 실패: %s", e)
        time.sleep(1.5)
    return new_count, total


def collect_listings() -> dict:
    naver_ok = True
    new_count = total = 0
    try:
        _fetch_molit_all()   # 실거래(합법)는 항상 우선 — 안전마진 baseline
    except Exception as e:
        logger.error("MOLIT 수집 오류(안전마진 baseline): %s", e)
    try:
        new_count, total = _fetch_naver_all()
    except requests.RequestException as e:
        naver_ok = False
        logger.warning("네이버 매물수집 skip(차단/오류) — 안전마진·실거래는 유지: %s", e)
    save_listing_collect_log(new_count, total, naver_ok)
    return {"new_count": new_count, "total_count": total, "naver_ok": naver_ok}