perf(travel-proxy): 배치 DB 연결 + nginx sync timeout 600s
- db.py: batch_sync_album, batch_mark_thumbs_done 추가 - indexer.py: 앨범 단위 배치 동기화로 전환 - nginx: /api/travel/ proxy_read_timeout 600s 추가 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -91,6 +91,7 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 600s;
|
||||
proxy_pass http://travel-proxy:8000/api/travel/;
|
||||
}
|
||||
|
||||
|
||||
@@ -175,3 +175,59 @@ def mark_thumb_done(album: str, filename: str) -> None:
|
||||
"UPDATE photos SET has_thumb = 1 WHERE album = ? AND filename = ?",
|
||||
(album, filename),
|
||||
)
|
||||
|
||||
|
||||
def batch_sync_album(album: str, items: List[Dict[str, Any]], existing_filenames: Set[str]) -> Dict[str, int]:
|
||||
"""앨범 단위 배치 동기화. 단일 커넥션으로 upsert + 삭제 처리."""
|
||||
added = updated = 0
|
||||
with _conn() as conn:
|
||||
for item in items:
|
||||
existing = conn.execute(
|
||||
"SELECT mtime FROM photos WHERE album = ? AND filename = ?",
|
||||
(album, item["filename"]),
|
||||
).fetchone()
|
||||
if not existing:
|
||||
conn.execute(
|
||||
"INSERT INTO photos (album, filename, mtime, has_thumb) VALUES (?, ?, ?, 0)",
|
||||
(album, item["filename"], item["mtime"]),
|
||||
)
|
||||
added += 1
|
||||
elif existing["mtime"] != item["mtime"]:
|
||||
conn.execute(
|
||||
"UPDATE photos SET mtime = ?, has_thumb = 0 WHERE album = ? AND filename = ?",
|
||||
(item["mtime"], album, item["filename"]),
|
||||
)
|
||||
updated += 1
|
||||
|
||||
# 삭제 처리
|
||||
db_rows = conn.execute(
|
||||
"SELECT filename FROM photos WHERE album = ?", (album,)
|
||||
).fetchall()
|
||||
db_filenames = {r["filename"] for r in db_rows}
|
||||
to_remove = db_filenames - existing_filenames
|
||||
|
||||
removed = len(to_remove)
|
||||
if to_remove:
|
||||
placeholders = ",".join("?" for _ in to_remove)
|
||||
conn.execute(
|
||||
f"DELETE FROM photos WHERE album = ? AND filename IN ({placeholders})",
|
||||
[album, *to_remove],
|
||||
)
|
||||
conn.execute(
|
||||
f"DELETE FROM album_covers WHERE album = ? AND filename IN ({placeholders})",
|
||||
[album, *to_remove],
|
||||
)
|
||||
|
||||
return {"added": added, "updated": updated, "removed": removed}
|
||||
|
||||
|
||||
def batch_mark_thumbs_done(items: List[Dict[str, str]]) -> None:
|
||||
"""썸네일 생성 완료 배치 표시."""
|
||||
if not items:
|
||||
return
|
||||
with _conn() as conn:
|
||||
for item in items:
|
||||
conn.execute(
|
||||
"UPDATE photos SET has_thumb = 1 WHERE album = ? AND filename = ?",
|
||||
(item["album"], item["filename"]),
|
||||
)
|
||||
|
||||
@@ -92,7 +92,7 @@ def sync(
|
||||
elif isinstance(v, dict) and isinstance(v.get("albums"), list):
|
||||
all_albums.update(v["albums"])
|
||||
|
||||
# 2. 각 앨범 폴더 스캔 → DB 동기화
|
||||
# 2. 각 앨범 폴더 스캔 → DB 배치 동기화
|
||||
added = 0
|
||||
updated = 0
|
||||
removed = 0
|
||||
@@ -100,29 +100,27 @@ def sync(
|
||||
for album in sorted(all_albums):
|
||||
folder = travel_root / album
|
||||
items = _scan_folder(folder)
|
||||
existing_filenames = set()
|
||||
existing_filenames = {item["filename"] for item in items}
|
||||
|
||||
for item in items:
|
||||
existing_filenames.add(item["filename"])
|
||||
result = db.upsert_photo(album, item["filename"], item["mtime"])
|
||||
if result == "added":
|
||||
added += 1
|
||||
elif result == "updated":
|
||||
updated += 1
|
||||
|
||||
removed += db.remove_missing_photos(album, existing_filenames)
|
||||
result = db.batch_sync_album(album, items, existing_filenames)
|
||||
added += result["added"]
|
||||
updated += result["updated"]
|
||||
removed += result["removed"]
|
||||
|
||||
# 3. 썸네일 미생성 분 일괄 생성
|
||||
no_thumb = db.get_photos_without_thumb()
|
||||
thumbs_generated = 0
|
||||
thumb_done_batch = []
|
||||
|
||||
for photo in no_thumb:
|
||||
src = travel_root / photo["album"] / photo["filename"]
|
||||
dest = thumb_root / photo["album"] / photo["filename"]
|
||||
if _generate_thumb(src, dest):
|
||||
db.mark_thumb_done(photo["album"], photo["filename"])
|
||||
thumb_done_batch.append(photo)
|
||||
thumbs_generated += 1
|
||||
|
||||
db.batch_mark_thumbs_done(thumb_done_batch)
|
||||
|
||||
duration = round(time.time() - start, 2)
|
||||
logger.info(
|
||||
"Sync complete: added=%d updated=%d removed=%d thumbs=%d duration=%.2fs",
|
||||
|
||||
Reference in New Issue
Block a user