feat(ai_news): pipeline uses articles_source (replaces Naver scraper)

stock-lab/app/screener/ai_news/pipeline.py

@@ -8,19 +8,17 @@ import logging
 import os
 import sqlite3
 import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
-import httpx
-
-from . import scraper as _scraper
+from . import scraper as _scraper  # legacy, kept for backward import
 from . import analyzer as _analyzer
+from . import articles_source  # 신규
 
 log = logging.getLogger(__name__)
 
 DEFAULT_TOP_N = 100
 DEFAULT_CONCURRENCY = 10
 DEFAULT_NEWS_PER_TICKER = 5
-DEFAULT_RATE_LIMIT_SEC = 0.2
 
 
 def _top_market_cap_tickers(conn: sqlite3.Connection, n: int) -> List[str]:
@@ -33,10 +31,6 @@ def _top_market_cap_tickers(conn: sqlite3.Connection, n: int) -> List[str]:
     return [r[0] for r in rows]
 
 
-def _make_http():
-    return httpx.AsyncClient(timeout=10.0, headers=_scraper.NAVER_HEADERS)
-
-
 def _make_llm():
     """Anthropic AsyncClient — env에 ANTHROPIC_API_KEY 필수."""
     from anthropic import AsyncAnthropic
@@ -44,47 +38,40 @@ def _make_llm():
 
 
 async def _process_one(
-    ticker: str, name: str, sem: asyncio.Semaphore,
-    http_client, llm, news_per_ticker: int, rate_limit_sec: float, model: str,
+    ticker: str, name: str, articles: List[Dict[str, Any]],
+    sem: asyncio.Semaphore, llm, model: str,
 ) -> Dict[str, Any]:
     async with sem:
-        if rate_limit_sec > 0:
-            await asyncio.sleep(rate_limit_sec)
-        news = await _scraper.fetch_news(http_client, ticker, n=news_per_ticker)
-        if not news:
-            return {
-                "ticker": ticker, "score_raw": 0.0, "reason": "no news",
-                "news_count": 0, "tokens_input": 0, "tokens_output": 0,
-                "model": model,
-            }
         return await _analyzer.score_sentiment(
-            llm, ticker, news, name=name, model=model,
+            llm, ticker, articles, name=name, model=model,
         )
 
 
 def _upsert_news_sentiment(
-    conn: sqlite3.Connection, asof: dt.date, rows: List[Dict[str, Any]]
+    conn: sqlite3.Connection, asof: dt.date,
+    rows: List[Dict[str, Any]], *, source: str = "articles",
 ) -> None:
     iso = asof.isoformat()
     data = [
         (
             r["ticker"], iso, r["score_raw"], r["reason"], r["news_count"],
-            r["tokens_input"], r["tokens_output"], r["model"],
+            r["tokens_input"], r["tokens_output"], r["model"], source,
         )
         for r in rows
     ]
     conn.executemany(
         """INSERT INTO news_sentiment
              (ticker, date, score_raw, reason, news_count,
-              tokens_input, tokens_output, model)
-           VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+              tokens_input, tokens_output, model, source)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(ticker, date) DO UPDATE SET
              score_raw=excluded.score_raw,
              reason=excluded.reason,
              news_count=excluded.news_count,
              tokens_input=excluded.tokens_input,
              tokens_output=excluded.tokens_output,
-             model=excluded.model
+             model=excluded.model,
+             source=excluded.source
         """,
         data,
     )
@@ -97,11 +84,10 @@ async def refresh_daily(
     *,
     top_n: int = DEFAULT_TOP_N,
     concurrency: int = DEFAULT_CONCURRENCY,
-    news_per_ticker: int = DEFAULT_NEWS_PER_TICKER,
-    rate_limit_sec: float = DEFAULT_RATE_LIMIT_SEC,
+    max_news_per_ticker: int = DEFAULT_NEWS_PER_TICKER,
+    window_days: int = 1,
     model: str = _analyzer.DEFAULT_MODEL,
 ) -> Dict[str, Any]:
-    """Returns summary dict with top_pos/top_neg/token totals/failures."""
     started = time.time()
     tickers = _top_market_cap_tickers(conn, n=top_n)
     name_map = {
@@ -111,16 +97,20 @@ async def refresh_daily(
         ).fetchall()
     } if tickers else {}
 
-    sem = asyncio.Semaphore(concurrency)
+    articles_by_ticker, mapping_stats = articles_source.gather_articles_for_tickers(
+        conn, tickers, asof,
+        window_days=window_days,
+        max_per_ticker=max_news_per_ticker,
+    )
 
-    async with _make_http() as http_client, _make_llm() as llm:
-        tasks = [
-            _process_one(
-                t, name_map.get(t, t), sem, http_client, llm,
-                news_per_ticker, rate_limit_sec, model,
-            )
-            for t in tickers
-        ]
+    sem = asyncio.Semaphore(concurrency)
+    async with _make_llm() as llm:
+        tasks = []
+        for t in tickers:
+            arts = articles_by_ticker.get(t, [])
+            if not arts:
+                continue  # 매핑 0 — score 미생성
+            tasks.append(_process_one(t, name_map.get(t, t), arts, sem, llm, model))
         raw_results = await asyncio.gather(*tasks, return_exceptions=True)
 
     successes: List[Dict[str, Any]] = []
@@ -132,7 +122,7 @@ async def refresh_daily(
             successes.append(r)
 
     if successes:
-        _upsert_news_sentiment(conn, asof, successes)
+        _upsert_news_sentiment(conn, asof, successes, source="articles")
 
     top_pos = sorted(successes, key=lambda r: -r["score_raw"])[:5]
     top_neg = sorted(successes, key=lambda r: r["score_raw"])[:5]
@@ -147,4 +137,5 @@ async def refresh_daily(
         "top_pos": top_pos,
         "top_neg": top_neg,
         "model": model,
+        "mapping": mapping_stats,
     }