60 lines
2.1 KiB
Python
60 lines
2.1 KiB
Python
import psutil
|
|
from datetime import datetime
|
|
|
|
|
|
class SystemMonitor:
|
|
def __init__(self, messenger, ollama_manager):
|
|
self.messenger = messenger
|
|
self.ollama_monitor = ollama_manager
|
|
self.last_health_check = datetime.now()
|
|
|
|
def check_health(self):
|
|
"""시스템 상태 점검 및 알림 (CPU, RAM, GPU) - 3분마다 실행"""
|
|
now = datetime.now()
|
|
if (now - self.last_health_check).total_seconds() < 180: # 5분 → 3분
|
|
return
|
|
|
|
self.last_health_check = now
|
|
alerts = []
|
|
|
|
# 1. CPU Check (non-blocking 측정)
|
|
cpu_usage = psutil.cpu_percent(interval=0)
|
|
|
|
if cpu_usage > 90:
|
|
# 검증: 상위 프로세스 CPU 합계 확인
|
|
top_processes = []
|
|
for proc in psutil.process_iter(['pid', 'name', 'cpu_percent']):
|
|
try:
|
|
if proc.info['name'] == 'System Idle Process':
|
|
continue
|
|
top_processes.append(proc.info)
|
|
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
|
pass
|
|
|
|
top_processes.sort(key=lambda x: x['cpu_percent'], reverse=True)
|
|
total_top_cpu = sum(p['cpu_percent'] for p in top_processes[:3])
|
|
|
|
if total_top_cpu >= 30.0:
|
|
top_3_str = ""
|
|
for p in top_processes[:3]:
|
|
top_3_str += f"\n- {p['name']} ({p['cpu_percent']}%)"
|
|
alerts.append(f"[CPU Overload] Usage: {cpu_usage}%\nTop Processes:{top_3_str}")
|
|
|
|
# 2. RAM Check
|
|
ram = psutil.virtual_memory()
|
|
if ram.percent > 90:
|
|
alerts.append(f"[RAM High] Usage: {ram.percent}% (Free: {ram.available / 1024**3:.1f}GB)")
|
|
|
|
# 3. GPU Check
|
|
if self.ollama_monitor:
|
|
gpu_status = self.ollama_monitor.get_gpu_status()
|
|
temp = gpu_status.get('temp', 0)
|
|
if temp > 80:
|
|
alerts.append(f"[GPU Overheat] Temp: {temp}C")
|
|
|
|
# 알림 전송
|
|
if alerts:
|
|
msg = "[System Health Alert]\n" + "\n".join(alerts)
|
|
if self.messenger:
|
|
self.messenger.send_message(msg)
|