"""
성과 평가 엔진 - PerformanceEvaluator
기능:
1. compute_metrics() - 핵심 성과 지표 계산
2. get_grade() - 지표별 S/A/B/C/D/F 등급 산출
3. generate_expert_panel() - Ollama LLM 5명 전문가 의견
4. generate_weekly_report() - 텔레그램 HTML 주간 보고서
"""
import json
import math
from datetime import datetime, timedelta
from modules.utils.performance_db import PerformanceDB
class PerformanceEvaluator:
def __init__(self):
self.perf_db = PerformanceDB()
# ─────────────────────────────────────────
# 1. 핵심 지표 계산
# ─────────────────────────────────────────
def compute_metrics(self, snapshots, trades):
"""성과 지표를 딕셔너리로 반환.
Args:
snapshots (list): daily_snapshots 리스트
trades (list): trade_records 리스트
Returns:
dict: 지표 딕셔너리 (또는 {"error": ...})
"""
if not snapshots:
return {"error": "스냅샷 데이터 없음 (운영 시작 후 첫 영업일까지 대기)"}
metrics = {}
# ── 수익률 ──────────────────────────────
initial = snapshots[0].get("total_eval", 0)
current = snapshots[-1].get("total_eval", 0)
metrics["total_return_pct"] = round(
(current - initial) / initial * 100, 2) if initial > 0 else 0.0
cutoff_7 = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
recent_snaps = [s for s in snapshots if s.get("date", "") >= cutoff_7]
if len(recent_snaps) >= 2:
w_init = recent_snaps[0].get("total_eval", 0)
w_curr = recent_snaps[-1].get("total_eval", 0)
metrics["weekly_return_pct"] = round(
(w_curr - w_init) / w_init * 100, 2) if w_init > 0 else 0.0
else:
metrics["weekly_return_pct"] = 0.0
# ── 리스크 지표 ──────────────────────────
daily_returns = [s.get("daily_return_pct", 0.0) / 100.0 for s in snapshots]
if len(daily_returns) >= 2:
mean_daily = sum(daily_returns) / len(daily_returns)
variance = sum((r - mean_daily) ** 2 for r in daily_returns) / len(daily_returns)
std_daily = math.sqrt(variance) if variance > 0 else 0.0
annual_return = mean_daily * 252
# Sharpe Ratio
if std_daily > 0:
metrics["sharpe_ratio"] = round(
annual_return / (std_daily * math.sqrt(252)), 3)
else:
metrics["sharpe_ratio"] = 0.0
# Sortino Ratio (하방 편차만 사용)
downside = [r for r in daily_returns if r < 0]
if downside:
dv = sum(r ** 2 for r in downside) / len(downside)
ds = math.sqrt(dv)
metrics["sortino_ratio"] = round(
annual_return / (ds * math.sqrt(252)), 3) if ds > 0 else 0.0
else:
metrics["sortino_ratio"] = 10.0 # 손실 없음
# Max Drawdown
peak = snapshots[0].get("total_eval", 0)
max_dd = 0.0
for snap in snapshots:
ev = snap.get("total_eval", 0)
if ev > peak:
peak = ev
if peak > 0:
dd = (peak - ev) / peak * 100
if dd > max_dd:
max_dd = dd
metrics["max_drawdown_pct"] = round(max_dd, 2)
# Calmar Ratio
ann_pct = annual_return * 100
metrics["calmar_ratio"] = round(
ann_pct / max_dd, 3) if max_dd > 0 else 0.0
else:
metrics["sharpe_ratio"] = 0.0
metrics["sortino_ratio"] = 0.0
metrics["max_drawdown_pct"] = 0.0
metrics["calmar_ratio"] = 0.0
# ── 매매 지표 ─────────────────────────────
closed = [t for t in trades
if t.get("action") == "BUY" and t.get("outcome_return_pct") is not None]
if closed:
wins = [t for t in closed if t.get("outcome_return_pct", 0) > 0]
losses = [t for t in closed if t.get("outcome_return_pct", 0) <= 0]
metrics["win_rate_pct"] = round(
len(wins) / len(closed) * 100, 1)
total_profit = sum(t["outcome_return_pct"] for t in wins)
total_loss = abs(sum(t["outcome_return_pct"] for t in losses))
metrics["profit_factor"] = round(
total_profit / total_loss, 3) if total_loss > 0 else 10.0
hd_list = [t["holding_days"] for t in closed if t.get("holding_days") is not None]
metrics["avg_holding_days"] = round(
sum(hd_list) / len(hd_list), 1) if hd_list else 0.0
else:
metrics["win_rate_pct"] = 0.0
metrics["profit_factor"] = 0.0
metrics["avg_holding_days"] = 0.0
metrics["total_trades"] = len(closed)
# ── 벤치마크 Alpha ────────────────────────
kospi_vals = [s.get("benchmark_kospi_close") for s in snapshots]
kospi_valid = [k for k in kospi_vals if k is not None]
if len(kospi_valid) >= 2:
kospi_ret = (kospi_valid[-1] - kospi_valid[0]) / kospi_valid[0] * 100
metrics["alpha"] = round(metrics["total_return_pct"] - kospi_ret, 2)
metrics["kospi_return_pct"] = round(kospi_ret, 2)
else:
metrics["alpha"] = 0.0
metrics["kospi_return_pct"] = 0.0
# ── AI 품질 지표 ──────────────────────────
if closed:
# LSTM 방향 정확도
correct = 0
direction_n = 0
for t in closed:
pred = t.get("ai_prediction_change")
outcome = t.get("outcome_return_pct")
if pred is not None and outcome is not None:
if (pred > 0) == (outcome > 0):
correct += 1
direction_n += 1
metrics["lstm_direction_accuracy"] = round(
correct / direction_n * 100, 1) if direction_n > 0 else 0.0
# 신호별 수익 상관도
outcomes = [t.get("outcome_return_pct", 0) for t in closed]
def pearson(xs, ys):
n = len(xs)
if n < 2:
return 0.0
mx = sum(xs) / n
my = sum(ys) / n
num = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
denom_x = sum((x - mx) ** 2 for x in xs)
denom_y = sum((y - my) ** 2 for y in ys)
denom = math.sqrt(denom_x * denom_y)
return num / denom if denom > 0 else 0.0
corr_tech = pearson([t.get("tech_score", 0) for t in closed], outcomes)
corr_sent = pearson([t.get("sentiment_score", 0) for t in closed], outcomes)
corr_lstm = pearson([t.get("lstm_score", 0) for t in closed], outcomes)
metrics["signal_correlation"] = {
"tech": round(corr_tech, 3),
"sentiment": round(corr_sent, 3),
"lstm": round(corr_lstm, 3)
}
metrics["best_signal_source"] = max(
["tech", "sentiment", "lstm"],
key=lambda k: abs(metrics["signal_correlation"][k])
)
else:
metrics["lstm_direction_accuracy"] = 0.0
metrics["signal_correlation"] = {"tech": 0.0, "sentiment": 0.0, "lstm": 0.0}
metrics["best_signal_source"] = "unknown"
metrics["snapshot_count"] = len(snapshots)
return metrics
# ─────────────────────────────────────────
# 2. 등급 산출
# ─────────────────────────────────────────
def get_grade(self, metric, value):
"""지표 이름과 값으로 S/A/B/C/D/F 등급 반환."""
# MDD는 낮을수록 좋음
if metric == "max_drawdown_pct":
thresholds = [(5, "S"), (10, "A"), (15, "B"), (20, "C"), (30, "D")]
for threshold, grade in thresholds:
if value < threshold:
return grade
return "F"
grade_rules = {
"sharpe_ratio": [(2.0, "S"), (1.5, "A"), (1.0, "B"), (0.5, "C"), (0.0, "D")],
"sortino_ratio": [(3.0, "S"), (2.0, "A"), (1.5, "B"), (1.0, "C"), (0.0, "D")],
"win_rate_pct": [(70, "S"), (60, "A"), (50, "B"), (40, "C"), (30, "D")],
"profit_factor": [(3.0, "S"), (2.0, "A"), (1.5, "B"), (1.0, "C"), (0.5, "D")],
"alpha": [(15, "S"), (10, "A"), (5, "B"), (0, "C"), (-5, "D")],
"total_return_pct": [(30, "S"), (20, "A"), (10, "B"), (0, "C"), (-10, "D")],
"weekly_return_pct": [(5, "S"), (3, "A"), (1, "B"), (0, "C"), (-1, "D")],
"lstm_direction_accuracy":[(70, "S"), (60, "A"), (55, "B"), (50, "C"), (40, "D")],
"calmar_ratio": [(3.0, "S"), (2.0, "A"), (1.0, "B"), (0.5, "C"), (0.0, "D")],
}
thresholds = grade_rules.get(metric, [])
for threshold, grade in thresholds:
if value >= threshold:
return grade
return "F"
# ─────────────────────────────────────────
# 3. 전문가 패널 (Ollama LLM)
# ─────────────────────────────────────────
def generate_expert_panel(self, metrics):
"""5명의 전문가 역할로 Ollama에 평가를 요청.
Returns:
list[dict]: [{role, grade, comment, suggestion}, ...]
"""
from modules.services.ollama import OllamaManager
ollama = OllamaManager()
sig_corr = metrics.get("signal_correlation", {})
experts = [
{
"role": "Risk Manager",
"focus": "risk level assessment and bankruptcy risk",
"data": (
f"Sharpe={metrics.get('sharpe_ratio', 0):.2f}, "
f"Sortino={metrics.get('sortino_ratio', 0):.2f}, "
f"MDD={metrics.get('max_drawdown_pct', 0):.1f}%, "
f"Calmar={metrics.get('calmar_ratio', 0):.2f}"
)
},
{
"role": "Fund Manager",
"focus": "alpha generation vs market benchmark",
"data": (
f"TotalReturn={metrics.get('total_return_pct', 0):.2f}%, "
f"Alpha={metrics.get('alpha', 0):.2f}%, "
f"KOSPI={metrics.get('kospi_return_pct', 0):.2f}%, "
f"WeeklyReturn={metrics.get('weekly_return_pct', 0):.2f}%"
)
},
{
"role": "Quant Analyst",
"focus": "AI model validity and signal quality",
"data": (
f"LSTM_Accuracy={metrics.get('lstm_direction_accuracy', 0):.1f}%, "
f"TechCorr={sig_corr.get('tech', 0):.3f}, "
f"SentCorr={sig_corr.get('sentiment', 0):.3f}, "
f"LSTMCorr={sig_corr.get('lstm', 0):.3f}, "
f"BestSignal={metrics.get('best_signal_source', 'N/A')}"
)
},
{
"role": "Trader",
"focus": "trading strategy effectiveness",
"data": (
f"WinRate={metrics.get('win_rate_pct', 0):.1f}%, "
f"ProfitFactor={metrics.get('profit_factor', 0):.2f}, "
f"AvgHolding={metrics.get('avg_holding_days', 0):.1f}days, "
f"TotalTrades={metrics.get('total_trades', 0)}"
)
},
{
"role": "Portfolio PM",
"focus": "overall strategy direction and sustainability",
"data": (
f"WeeklyReturn={metrics.get('weekly_return_pct', 0):.2f}%, "
f"Sharpe={metrics.get('sharpe_ratio', 0):.2f}, "
f"WinRate={metrics.get('win_rate_pct', 0):.1f}%, "
f"Alpha={metrics.get('alpha', 0):.2f}%, "
f"MDD={metrics.get('max_drawdown_pct', 0):.1f}%"
)
}
]
results = []
for exp in experts:
prompt = (
f"You are a professional {exp['role']} evaluating an AI stock trading bot. "
f"Your focus: {exp['focus']}. "
f"Performance data: {exp['data']}. "
f"Respond ONLY with valid JSON (no markdown, no extra text): "
f"{{\"grade\":\"S|A|B|C|D|F\","
f"\"comment\":\"1 sentence evaluation in Korean\","
f"\"suggestion\":\"1 sentence improvement tip in Korean\"}}"
)
try:
resp = ollama.request_inference(prompt)
if not resp:
raise ValueError("Empty response from Ollama")
data = json.loads(resp)
results.append({
"role": exp["role"],
"grade": data.get("grade", "C"),
"comment": data.get("comment", "(응답 없음)"),
"suggestion": data.get("suggestion", "데이터 축적 필요")
})
except Exception as e:
print(f"[Evaluator] Expert panel [{exp['role']}] error: {e}")
results.append({
"role": exp["role"],
"grade": "C",
"comment": "평가 데이터가 부족합니다.",
"suggestion": "더 많은 거래 데이터 축적 후 재평가를 권장합니다."
})
return results
# ─────────────────────────────────────────
# 4. 주간 보고서 생성
# ─────────────────────────────────────────
def generate_weekly_report(self):
"""주간 성과 보고서 (텔레그램 HTML 형식) 반환."""
snapshots = self.perf_db.load_snapshots(days=7)
# 매매 완료 건은 30일치 사용 (주간 거래 수가 적을 수 있음)
trades = self.perf_db.load_trades(days=30)
metrics = self.compute_metrics(snapshots, trades)
if "error" in metrics:
return (
f"[주간 성과 평가 보고서]\n"
f"⚠️ {metrics['error']}\n"
f"매일 오전 09:05~09:15에 스냅샷이 저장됩니다."
)
# 등급 계산
g_sharpe = self.get_grade("sharpe_ratio", metrics.get("sharpe_ratio", 0))
g_win = self.get_grade("win_rate_pct", metrics.get("win_rate_pct", 0))
g_mdd = self.get_grade("max_drawdown_pct", metrics.get("max_drawdown_pct", 0))
g_alpha = self.get_grade("alpha", metrics.get("alpha", 0))
g_weekly = self.get_grade("weekly_return_pct", metrics.get("weekly_return_pct", 0))
g_lstm = self.get_grade("lstm_direction_accuracy",
metrics.get("lstm_direction_accuracy", 0))
# 종합 등급 (Sharpe/Win/MDD/Alpha 평균)
grade_map = {"S": 5, "A": 4, "B": 3, "C": 2, "D": 1, "F": 0}
grade_rev = {v: k for k, v in grade_map.items()}
key_grades = [grade_map[g] for g in [g_sharpe, g_win, g_mdd, g_alpha]]
overall_grade = grade_rev[round(sum(key_grades) / len(key_grades))]
# 전문가 패널 (Ollama 호출)
try:
experts = self.generate_expert_panel(metrics)
except Exception as e:
print(f"[Evaluator] Expert panel skipped: {e}")
experts = []
now_str = datetime.now().strftime("%Y/%m/%d %H:%M")
corr = metrics.get("signal_correlation", {})
report = (
f"📊 [주간 성과 평가 보고서] {now_str}\n"
f"━━━━━━━━━━━━━━━━━━━━━━\n"
f"\n■ 수익률\n"
f" 주간: {metrics.get('weekly_return_pct', 0):+.2f}% [{g_weekly}]"
f" 누적: {metrics.get('total_return_pct', 0):+.2f}%\n"
f" Alpha: {metrics.get('alpha', 0):+.2f}% [{g_alpha}]"
f" vs KOSPI {metrics.get('kospi_return_pct', 0):+.2f}%\n"
f"\n■ 리스크\n"
f" Sharpe: {metrics.get('sharpe_ratio', 0):.2f} [{g_sharpe}]"
f" Sortino: {metrics.get('sortino_ratio', 0):.2f}\n"
f" MDD: {metrics.get('max_drawdown_pct', 0):.1f}% [{g_mdd}]"
f" Calmar: {metrics.get('calmar_ratio', 0):.2f}\n"
f"\n■ 매매 통계\n"
f" 승률: {metrics.get('win_rate_pct', 0):.1f}% [{g_win}]"
f" PF: {metrics.get('profit_factor', 0):.2f}\n"
f" 평균보유: {metrics.get('avg_holding_days', 0):.1f}일"
f" 완료매매: {metrics.get('total_trades', 0)}건\n"
f"\n■ AI 품질\n"
f" LSTM 방향정확도: {metrics.get('lstm_direction_accuracy', 0):.1f}%"
f" [{g_lstm}]\n"
f" 신호 상관도 — Tech: {corr.get('tech', 0):.3f}"
f" Sent: {corr.get('sentiment', 0):.3f}"
f" LSTM: {corr.get('lstm', 0):.3f}\n"
f" 최고기여 신호: {metrics.get('best_signal_source', 'N/A')}\n"
)
if experts:
role_icons = {
"Risk Manager": "🛡",
"Fund Manager": "💼",
"Quant Analyst": "🧮",
"Trader": "📈",
"Portfolio PM": "🏦"
}
report += "\n■ 전문가 패널 의견\n"
for exp in experts:
icon = role_icons.get(exp["role"], "👤")
report += (
f"{icon} {exp['role']} [{exp['grade']}]\n"
f" {exp['comment']}\n"
f" 💡 {exp['suggestion']}\n"
)
report += (
f"\n━━━━━━━━━━━━━━━━━━━━━━\n"
f"🏆 종합 등급: [{overall_grade}]\n"
f"스냅샷 {metrics.get('snapshot_count', 0)}일 | 완료매매 {metrics.get('total_trades', 0)}건 기준"
)
return report