gahusb
cdc309150e
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EqCYBhvTcdeCTUDX3RhWx9
188 lines
8.3 KiB
Python
188 lines
8.3 KiB
Python
"""매물 수집: 국토부 실거래가(합법, market_deals) + 네이버부동산 호가(회색·폴백, listings)."""
|
||
import os
|
||
import re
|
||
import time
|
||
import logging
|
||
import requests
|
||
from datetime import date
|
||
|
||
from .lawd_codes import LAWD_CODES
|
||
from .db import (get_listing_criteria, upsert_listing, upsert_market_deal,
|
||
save_listing_collect_log)
|
||
|
||
logger = logging.getLogger("realestate-lab")
|
||
|
||
MOLIT_KEY = os.getenv("DATA_GO_KR_API_KEY", "")
|
||
NAVER_BASE = "https://new.land.naver.com/api"
|
||
# MOLIT operation ID (스펙 §3, 라이브 확인 필요). house_type → (rent_op, trade_op)
|
||
MOLIT_OPS = {
|
||
"아파트": ("getRTMSDataSvcAptRent", "getRTMSDataSvcAptTradeDev"),
|
||
"오피스텔": ("getRTMSDataSvcOffiRent", "getRTMSDataSvcOffiTrade"),
|
||
"연립다세대": ("getRTMSDataSvcRHRent", "getRTMSDataSvcRHTrade"),
|
||
}
|
||
_HOUSE_TYPE_ALIAS = {"빌라": "연립다세대"}
|
||
|
||
|
||
def _won_to_manwon(s) -> int | None:
|
||
"""'43,000' 또는 '2억 9,000' → 만원 정수."""
|
||
if s is None:
|
||
return None
|
||
t = str(s).replace(",", "").strip()
|
||
if not t:
|
||
return None
|
||
m = re.match(r"(?:(\d+)억)?\s*(\d+)?", t)
|
||
if m and (m.group(1) or m.group(2)):
|
||
eok = int(m.group(1)) if m.group(1) else 0
|
||
man = int(m.group(2)) if m.group(2) else 0
|
||
return eok * 10000 + man
|
||
try:
|
||
return int(float(t))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _parse_molit_rent(raw: dict, house_type: str) -> dict:
|
||
deposit = _won_to_manwon(raw.get("deposit"))
|
||
rent = _won_to_manwon(raw.get("monthlyRent"))
|
||
return {"house_type": house_type, "complex_name": (raw.get("aptNm") or raw.get("offiNm") or "").strip(),
|
||
"area": float(raw.get("excluUseAr") or 0) or None, "deal_type": "전세" if not rent else "월세",
|
||
"deposit": deposit, "monthly_rent": rent, "sale_amount": None,
|
||
"deal_ym": f"{raw.get('dealYear','')}{str(raw.get('dealMonth','')).zfill(2)}",
|
||
"floor": str(raw.get("floor") or ""), "source": "molit"}
|
||
|
||
|
||
def _parse_molit_trade(raw: dict, house_type: str) -> dict:
|
||
return {"house_type": house_type, "complex_name": (raw.get("aptNm") or raw.get("offiNm") or "").strip(),
|
||
"area": float(raw.get("excluUseAr") or 0) or None, "deal_type": "매매",
|
||
"deposit": None, "monthly_rent": None,
|
||
"sale_amount": _won_to_manwon(raw.get("dealAmount")),
|
||
"deal_ym": f"{raw.get('dealYear','')}{str(raw.get('dealMonth','')).zfill(2)}",
|
||
"floor": str(raw.get("floor") or ""), "source": "molit"}
|
||
|
||
|
||
def _parse_naver_article(raw: dict) -> dict:
|
||
dt = raw.get("tradeTypeName") or ""
|
||
prc = _won_to_manwon(raw.get("dealOrWarrantPrc"))
|
||
rent = _won_to_manwon(raw.get("rentPrc"))
|
||
is_sale = dt == "매매"
|
||
return {"article_no": str(raw.get("articleNo") or ""), "source": "naver", "deal_type": dt,
|
||
"deposit": None if is_sale else prc, "monthly_rent": rent,
|
||
"sale_price": prc if is_sale else None,
|
||
"area_exclusive": float(raw.get("area2") or raw.get("areaName") or 0) or None,
|
||
"floor": raw.get("floorInfo"), "complex_name": raw.get("articleName") or raw.get("buildingName"),
|
||
"dong_code": (raw.get("cortarNo") or "")[:5] or None,
|
||
"url": f"https://new.land.naver.com/houses?articleNo={raw.get('articleNo')}",
|
||
"raw_json": None}
|
||
|
||
|
||
def _molit_call(op: str, lawd: str, ym: str):
|
||
"""국토부 실거래 단건 호출 (재시도). Returns (items, diag).
|
||
diag=None on success, else 짧은 진단 문자열(http=401 / json_err / req_err=...) — 관측용."""
|
||
if not MOLIT_KEY:
|
||
return [], "no_key"
|
||
url = f"https://apis.data.go.kr/1613000/RTMSDataSvc{op[len('getRTMSDataSvc'):]}/{op}"
|
||
last_diag = None
|
||
for attempt in range(3):
|
||
try:
|
||
resp = requests.get(url, params={"serviceKey": MOLIT_KEY, "LAWD_CD": lawd,
|
||
"DEAL_YMD": ym, "numOfRows": 200, "_type": "json"}, timeout=30)
|
||
resp.raise_for_status()
|
||
body = resp.json()
|
||
items = (((body.get("response") or {}).get("body") or {}).get("items") or {}).get("item") or []
|
||
return (items if isinstance(items, list) else [items]), None
|
||
except requests.HTTPError as e:
|
||
code = getattr(getattr(e, "response", None), "status_code", "?")
|
||
last_diag = f"http={code}"
|
||
except requests.RequestException as e:
|
||
last_diag = f"req_err={type(e).__name__}"
|
||
except ValueError:
|
||
# 200이지만 JSON 아님(XML 오류 응답 등) — 재시도 무의미
|
||
last_diag = "json_err"
|
||
break
|
||
if attempt < 2:
|
||
time.sleep(2 ** attempt)
|
||
logger.error("MOLIT %s 실패: %s", op, last_diag)
|
||
return [], last_diag
|
||
|
||
|
||
def _fetch_molit_all():
|
||
"""대상 동 시군구코드 × 유형 × 최근 2개월 전월세·매매 실거래 수집→market_deals.
|
||
Returns (saved_count, diag). diag = 'calls=N saved=M [err=<첫에러>]' 관측용."""
|
||
saved = 0
|
||
calls = 0
|
||
first_err = None
|
||
lawds = set(LAWD_CODES.values())
|
||
today = date.today()
|
||
yms = [today.strftime("%Y%m"),
|
||
(today.replace(day=1) - __import__("datetime").timedelta(days=1)).strftime("%Y%m")]
|
||
for lawd in lawds:
|
||
for htype, (rent_op, trade_op) in MOLIT_OPS.items():
|
||
for ym in yms:
|
||
for op, parser in ((rent_op, _parse_molit_rent), (trade_op, _parse_molit_trade)):
|
||
items, diag = _molit_call(op, lawd, ym)
|
||
calls += 1
|
||
if diag and first_err is None:
|
||
first_err = diag
|
||
for raw in items:
|
||
try:
|
||
d = parser(raw, htype)
|
||
d["dong_code"] = lawd
|
||
d["dong"] = (raw.get("umdNm") or "").strip() or None
|
||
upsert_market_deal(d)
|
||
saved += 1
|
||
except Exception as e:
|
||
logger.warning("MOLIT 파싱 실패: %s", e)
|
||
time.sleep(0.3)
|
||
diag = f"calls={calls} saved={saved}" + (f" err={first_err}" if first_err else "")
|
||
return saved, diag
|
||
|
||
|
||
def _fetch_naver_all() -> tuple:
|
||
"""대상 동 네이버 호가 매물 수집→listings. (new_count, total_count) 반환. 예외는 상위로 전파."""
|
||
new_count = total = 0
|
||
crit = get_listing_criteria()
|
||
for dong in crit.get("dongs", []):
|
||
code = LAWD_CODES.get(dong)
|
||
if not code:
|
||
continue
|
||
# 저빈도: 동당 1회, UA/Referer 현실값. 응답 articleList 순회.
|
||
resp = requests.get(f"{NAVER_BASE}/articles",
|
||
params={"cortarNo": code, "order": "dateDesc", "page": 1},
|
||
headers={"User-Agent": "Mozilla/5.0", "Referer": "https://new.land.naver.com/"},
|
||
timeout=20)
|
||
resp.raise_for_status()
|
||
for raw in (resp.json().get("articleList") or []):
|
||
try:
|
||
d = _parse_naver_article(raw)
|
||
if not d["article_no"]:
|
||
continue
|
||
d["dong"] = dong
|
||
_, is_new = upsert_listing(d)
|
||
total += 1
|
||
if is_new:
|
||
new_count += 1
|
||
except Exception as e:
|
||
logger.warning("네이버 파싱 실패: %s", e)
|
||
time.sleep(1.5)
|
||
return new_count, total
|
||
|
||
|
||
def collect_listings() -> dict:
|
||
naver_ok = True
|
||
new_count = total = 0
|
||
molit_diag = naver_diag = None
|
||
try:
|
||
_, molit_diag = _fetch_molit_all() # 실거래(합법)는 항상 우선 — 안전마진 baseline
|
||
except Exception as e:
|
||
molit_diag = f"exc={type(e).__name__}:{str(e)[:80]}"
|
||
logger.error("MOLIT 수집 오류(안전마진 baseline): %s", e)
|
||
try:
|
||
new_count, total = _fetch_naver_all()
|
||
except requests.RequestException as e:
|
||
naver_ok = False
|
||
naver_diag = f"{type(e).__name__}:{str(e)[:80]}"
|
||
logger.warning("네이버 매물수집 skip(차단/오류) — 안전마진·실거래는 유지: %s", e)
|
||
diag = f"molit[{molit_diag}] naver[{naver_diag or 'ok'}]"
|
||
save_listing_collect_log(new_count, total, naver_ok, error=diag)
|
||
return {"new_count": new_count, "total_count": total, "naver_ok": naver_ok, "diag": diag}
|