perf(travel-proxy): 배치 DB 연결 + nginx sync timeout 600s
- db.py: batch_sync_album, batch_mark_thumbs_done 추가 - indexer.py: 앨범 단위 배치 동기화로 전환 - nginx: /api/travel/ proxy_read_timeout 600s 추가 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -91,6 +91,7 @@ server {
|
|||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
proxy_pass http://travel-proxy:8000/api/travel/;
|
proxy_pass http://travel-proxy:8000/api/travel/;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -175,3 +175,59 @@ def mark_thumb_done(album: str, filename: str) -> None:
|
|||||||
"UPDATE photos SET has_thumb = 1 WHERE album = ? AND filename = ?",
|
"UPDATE photos SET has_thumb = 1 WHERE album = ? AND filename = ?",
|
||||||
(album, filename),
|
(album, filename),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def batch_sync_album(album: str, items: List[Dict[str, Any]], existing_filenames: Set[str]) -> Dict[str, int]:
|
||||||
|
"""앨범 단위 배치 동기화. 단일 커넥션으로 upsert + 삭제 처리."""
|
||||||
|
added = updated = 0
|
||||||
|
with _conn() as conn:
|
||||||
|
for item in items:
|
||||||
|
existing = conn.execute(
|
||||||
|
"SELECT mtime FROM photos WHERE album = ? AND filename = ?",
|
||||||
|
(album, item["filename"]),
|
||||||
|
).fetchone()
|
||||||
|
if not existing:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO photos (album, filename, mtime, has_thumb) VALUES (?, ?, ?, 0)",
|
||||||
|
(album, item["filename"], item["mtime"]),
|
||||||
|
)
|
||||||
|
added += 1
|
||||||
|
elif existing["mtime"] != item["mtime"]:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE photos SET mtime = ?, has_thumb = 0 WHERE album = ? AND filename = ?",
|
||||||
|
(item["mtime"], album, item["filename"]),
|
||||||
|
)
|
||||||
|
updated += 1
|
||||||
|
|
||||||
|
# 삭제 처리
|
||||||
|
db_rows = conn.execute(
|
||||||
|
"SELECT filename FROM photos WHERE album = ?", (album,)
|
||||||
|
).fetchall()
|
||||||
|
db_filenames = {r["filename"] for r in db_rows}
|
||||||
|
to_remove = db_filenames - existing_filenames
|
||||||
|
|
||||||
|
removed = len(to_remove)
|
||||||
|
if to_remove:
|
||||||
|
placeholders = ",".join("?" for _ in to_remove)
|
||||||
|
conn.execute(
|
||||||
|
f"DELETE FROM photos WHERE album = ? AND filename IN ({placeholders})",
|
||||||
|
[album, *to_remove],
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
f"DELETE FROM album_covers WHERE album = ? AND filename IN ({placeholders})",
|
||||||
|
[album, *to_remove],
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"added": added, "updated": updated, "removed": removed}
|
||||||
|
|
||||||
|
|
||||||
|
def batch_mark_thumbs_done(items: List[Dict[str, str]]) -> None:
|
||||||
|
"""썸네일 생성 완료 배치 표시."""
|
||||||
|
if not items:
|
||||||
|
return
|
||||||
|
with _conn() as conn:
|
||||||
|
for item in items:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE photos SET has_thumb = 1 WHERE album = ? AND filename = ?",
|
||||||
|
(item["album"], item["filename"]),
|
||||||
|
)
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ def sync(
|
|||||||
elif isinstance(v, dict) and isinstance(v.get("albums"), list):
|
elif isinstance(v, dict) and isinstance(v.get("albums"), list):
|
||||||
all_albums.update(v["albums"])
|
all_albums.update(v["albums"])
|
||||||
|
|
||||||
# 2. 각 앨범 폴더 스캔 → DB 동기화
|
# 2. 각 앨범 폴더 스캔 → DB 배치 동기화
|
||||||
added = 0
|
added = 0
|
||||||
updated = 0
|
updated = 0
|
||||||
removed = 0
|
removed = 0
|
||||||
@@ -100,29 +100,27 @@ def sync(
|
|||||||
for album in sorted(all_albums):
|
for album in sorted(all_albums):
|
||||||
folder = travel_root / album
|
folder = travel_root / album
|
||||||
items = _scan_folder(folder)
|
items = _scan_folder(folder)
|
||||||
existing_filenames = set()
|
existing_filenames = {item["filename"] for item in items}
|
||||||
|
|
||||||
for item in items:
|
result = db.batch_sync_album(album, items, existing_filenames)
|
||||||
existing_filenames.add(item["filename"])
|
added += result["added"]
|
||||||
result = db.upsert_photo(album, item["filename"], item["mtime"])
|
updated += result["updated"]
|
||||||
if result == "added":
|
removed += result["removed"]
|
||||||
added += 1
|
|
||||||
elif result == "updated":
|
|
||||||
updated += 1
|
|
||||||
|
|
||||||
removed += db.remove_missing_photos(album, existing_filenames)
|
|
||||||
|
|
||||||
# 3. 썸네일 미생성 분 일괄 생성
|
# 3. 썸네일 미생성 분 일괄 생성
|
||||||
no_thumb = db.get_photos_without_thumb()
|
no_thumb = db.get_photos_without_thumb()
|
||||||
thumbs_generated = 0
|
thumbs_generated = 0
|
||||||
|
thumb_done_batch = []
|
||||||
|
|
||||||
for photo in no_thumb:
|
for photo in no_thumb:
|
||||||
src = travel_root / photo["album"] / photo["filename"]
|
src = travel_root / photo["album"] / photo["filename"]
|
||||||
dest = thumb_root / photo["album"] / photo["filename"]
|
dest = thumb_root / photo["album"] / photo["filename"]
|
||||||
if _generate_thumb(src, dest):
|
if _generate_thumb(src, dest):
|
||||||
db.mark_thumb_done(photo["album"], photo["filename"])
|
thumb_done_batch.append(photo)
|
||||||
thumbs_generated += 1
|
thumbs_generated += 1
|
||||||
|
|
||||||
|
db.batch_mark_thumbs_done(thumb_done_batch)
|
||||||
|
|
||||||
duration = round(time.time() - start, 2)
|
duration = round(time.time() - start, 2)
|
||||||
logger.info(
|
logger.info(
|
||||||
"Sync complete: added=%d updated=%d removed=%d thumbs=%d duration=%.2fs",
|
"Sync complete: added=%d updated=%d removed=%d thumbs=%d duration=%.2fs",
|
||||||
|
|||||||
Reference in New Issue
Block a user