fix(naver-fetch): realestate base :18800 + 동별 ingest + 180s no-retry
This commit is contained in:
@@ -58,12 +58,14 @@ async def run_cycle(nas, naver, state, settings) -> None:
|
||||
|
||||
dongs = tg.get("dongs", [])
|
||||
page_limit = int(tg.get("page_limit", 1))
|
||||
batches: list[dict] = []
|
||||
fetched_at = _iso_utc_now()
|
||||
# 동별로 fetch → 즉시 ingest. 전체 batch를 한 POST로 보내면 NAS 동기 처리(매물 수 비례)가
|
||||
# >180s로 타임아웃하므로 동 단위로 쪼갬(각 ~수십건 → 빠름, 부분 실패 격리). batches 리스트 계약 호환.
|
||||
for d in dongs:
|
||||
cortar = d.get("cortar_no")
|
||||
dong = d.get("dong")
|
||||
articles: list[dict] = []
|
||||
try:
|
||||
articles: list[dict] = []
|
||||
for page in range(1, page_limit + 1):
|
||||
resp = await naver.fetch_articles(cortar, page)
|
||||
articles += resp.get("articleList", [])
|
||||
@@ -72,20 +74,16 @@ async def run_cycle(nas, naver, state, settings) -> None:
|
||||
except Exception:
|
||||
logger.exception("naver fetch 실패 dong=%s", dong)
|
||||
state.errors += 1
|
||||
batches.append({"dong": dong, "articles": articles})
|
||||
continue
|
||||
try:
|
||||
r = await nas.post_ingest(fetched_at, [{"dong": dong, "articles": articles}])
|
||||
logger.info("ingest dong=%s received=%s new=%s matched=%s",
|
||||
dong, r.get("received"), r.get("new"), r.get("matched"))
|
||||
state.listings_pushed += len(articles)
|
||||
except Exception:
|
||||
logger.exception("listings-ingest 실패 dong=%s", dong)
|
||||
state.errors += 1
|
||||
|
||||
fetched_at = _iso_utc_now()
|
||||
try:
|
||||
r = await nas.post_ingest(fetched_at, batches)
|
||||
logger.info("ingest received=%s new=%s matched=%s",
|
||||
r.get("received"), r.get("new"), r.get("matched"))
|
||||
except Exception:
|
||||
logger.exception("listings-ingest 실패")
|
||||
state.errors += 1
|
||||
state.state = "idle"
|
||||
return
|
||||
|
||||
state.listings_pushed += sum(len(b["articles"]) for b in batches)
|
||||
state.last_fetch_at = _epoch_now()
|
||||
state.state = "idle"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user