feat(realestate-collector): 30-day window + district extraction + completed skip

- Add _extract_district() helper with DISTRICT_PATTERN regex (서울 only)
- collect_all() now passes RCRIT_PBLANC_DE_FROM param (30-day window) to all detail endpoints
- collect_all() skips announcements where compute_status() returns '완료'
- collect_all() stamps district on each parsed announcement before upsert
- upsert_announcement(): add district to INSERT/VALUES/ON CONFLICT UPDATE; data.setdefault('district', None)
- ANNOUNCEMENT_COLUMNS: add 'district' (closes deferred gap from Task 2 review)
- 9 new tests in realestate-lab/tests/test_collector.py (6 unit + 3 integration)
- Full suite: 22 passed

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-28 08:28:10 +09:00
parent 496e3a6a73
commit 9dd517e82a
3 changed files with 178 additions and 9 deletions

View File

@@ -1,9 +1,11 @@
import os
import re
import logging
from datetime import date, timedelta
import requests
from typing import List, Dict, Any
from .db import upsert_announcement, upsert_model, save_collect_log
from .db import upsert_announcement, upsert_model, save_collect_log, compute_status
logger = logging.getLogger("realestate-lab")
@@ -19,6 +21,19 @@ DETAIL_ENDPOINTS = [
("getOPTLttotPblancDetail", "getOPTLttotPblancMdl"),
]
DISTRICT_PATTERN = re.compile(r"(?:서울특별시|서울시|서울)\s+(\S+?(?:구|군))")
def _extract_district(parsed: Dict[str, Any]) -> str | None:
"""파싱된 공고에서 자치구를 추출. 서울 외 지역·실패 시 None."""
for src in (parsed.get("address"), parsed.get("region_name")):
if not src:
continue
m = DISTRICT_PATTERN.search(src)
if m:
return m.group(1)
return None
def _api_call(endpoint: str, params: Dict[str, Any] = None) -> List[Dict]:
"""페이지네이션 처리하여 API 전체 데이터를 반환한다."""
@@ -130,28 +145,49 @@ def _parse_model(raw: Dict[str, Any]) -> Dict[str, Any]:
def collect_all() -> Dict[str, Any]:
"""모든 엔드포인트를 순회하며 공고 + 모델 데이터를 수집·저장한다."""
"""모든 엔드포인트를 순회하며 공고 + 모델 데이터를 수집·저장한다.
모집공고일 30일 이전 데이터는 API 파라미터로 사전 좁힘.
status='완료'로 판정되는 응답은 저장하지 않음.
"""
if not API_KEY:
logger.warning("API 키 미설정 — 수집 중단")
save_collect_log(0, 0, "API 키 미설정")
return {"new_count": 0, "total_count": 0}
today = date.today()
date_from = (today - timedelta(days=30)).strftime("%Y%m%d")
total_count = 0
new_count = 0
skipped_completed = 0
for detail_ep, model_ep in DETAIL_ENDPOINTS:
# 공고 상세 수집
detail_rows = _api_call(detail_ep)
# 공고 상세 수집 — API에 모집공고일 윈도우 파라미터 전달
# 일부 엔드포인트는 파라미터 미지원일 수 있어 무시되지만 응답에 영향 없음
detail_rows = _api_call(detail_ep, params={"RCRIT_PBLANC_DE_FROM": date_from})
for raw in detail_rows:
try:
parsed = _parse_apt_detail(raw)
# 일정 정보가 하나도 없는 공고는 건너뜀
parsed["district"] = _extract_district(parsed)
# 일정 정보가 하나도 없는 공고는 건너뜀 (기존)
has_dates = any(parsed.get(f) for f in (
"receipt_start", "receipt_end", "spsply_start",
"gnrl_rank1_start", "winner_date", "contract_start",
))
if not has_dates:
continue
# status='완료'면 저장하지 않음 (자원 절감)
status = compute_status(
parsed.get("receipt_start", "") or "",
parsed.get("receipt_end", "") or "",
parsed.get("winner_date", "") or "",
)
if status == "완료":
skipped_completed += 1
continue
_, is_new = upsert_announcement(parsed)
total_count += 1
if is_new:
@@ -168,5 +204,5 @@ def collect_all() -> Dict[str, Any]:
except Exception as e:
logger.error("모델 upsert 실패 [%s]: %s", model_ep, e)
save_collect_log(new_count, total_count)
logger.info("수집 완료: new=%d, total=%d", new_count, total_count)
logger.info("수집 완료: new=%d, total=%d, skipped_completed=%d", new_count, total_count, skipped_completed)
return {"new_count": new_count, "total_count": total_count}