264 lines
12 KiB
Python
264 lines
12 KiB
Python
"""Графики для презентации: матрицы ошибок, сравнение моделей, ошибки по классам."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import matplotlib
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
import pandas as pd
|
|
from matplotlib.colors import LinearSegmentedColormap
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / "presentation" / "generated"
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
OOF = ROOT / "results" / "final" / "finalists" / "voting_et_char_lean" / "oof_predictions.csv"
|
|
METRICS = ROOT / "FINAL_RESULTS.csv"
|
|
|
|
plt.rcParams.update({
|
|
"font.family": "DejaVu Sans",
|
|
"font.size": 13,
|
|
"axes.titlesize": 18,
|
|
"axes.labelsize": 13,
|
|
"savefig.facecolor": "#F8F1DC",
|
|
"figure.facecolor": "#F8F1DC",
|
|
})
|
|
|
|
CREAM = "#F8F1DC"
|
|
DARK = "#203A2B"
|
|
GREEN = "#4D7C4A"
|
|
BROWN = "#8B5A2B"
|
|
RED = "#B23A2E"
|
|
GOLD = "#D8A23A"
|
|
|
|
CMAP = LinearSegmentedColormap.from_list("stardew_blue", ["#FFFDF4", "#BFD9C0", "#2E6B57", "#173A34"])
|
|
|
|
|
|
def load_oof() -> pd.DataFrame:
|
|
frame = pd.read_csv(OOF)
|
|
frame["y_true"] = frame["y_true"].astype(str)
|
|
frame["y_pred"] = frame["y_pred"].astype(str)
|
|
return frame
|
|
|
|
|
|
def top_confusions(frame: pd.DataFrame, n: int = 18) -> pd.Series:
|
|
errors = frame[frame["y_true"] != frame["y_pred"]]
|
|
pairs = errors.groupby(["y_true", "y_pred"]).size().sort_values(ascending=False).head(n)
|
|
pairs.index = [f"{a} → {b}" for a, b in pairs.index]
|
|
return pairs
|
|
|
|
|
|
def confusion_top12(frame: pd.DataFrame) -> None:
|
|
labels = frame["y_true"].value_counts().head(12).index.tolist()
|
|
matrix = pd.crosstab(frame["y_true"], frame["y_pred"]).reindex(index=labels, fill_value=0)
|
|
shown = matrix[[c for c in labels if c in matrix.columns]].copy()
|
|
shown["прочие"] = matrix.drop(columns=[c for c in labels if c in matrix.columns]).sum(axis=1)
|
|
shown = shown.div(shown.sum(axis=1), axis=0) * 100
|
|
|
|
fig, ax = plt.subplots(figsize=(12.5, 7.4))
|
|
image = ax.imshow(shown.to_numpy(), cmap=CMAP, vmin=0, vmax=60, aspect="auto")
|
|
ax.set_xticks(np.arange(shown.shape[1]))
|
|
ax.set_yticks(np.arange(shown.shape[0]))
|
|
ax.set_xticklabels(shown.columns, rotation=42, ha="right", fontsize=10)
|
|
ax.set_yticklabels(shown.index, fontsize=11)
|
|
ax.set_xlabel("Предсказанный персонаж", fontsize=14, color=DARK)
|
|
ax.set_ylabel("Истинный персонаж", fontsize=14, color=DARK)
|
|
ax.set_title("Матрица ошибок: 12 самых частых персонажей, % строки", fontsize=18, color=DARK, pad=12)
|
|
for row in range(shown.shape[0]):
|
|
for col in range(shown.shape[1]):
|
|
value = shown.iloc[row, col]
|
|
if value >= 0.5:
|
|
color = "white" if value > 32 else DARK
|
|
ax.text(col, row, f"{value:.0f}", ha="center", va="center", fontsize=9, color=color)
|
|
cbar = fig.colorbar(image, ax=ax, fraction=0.035, pad=0.02)
|
|
cbar.set_label("% строки", color=DARK)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "confusion_top12.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def confusion_full(frame: pd.DataFrame) -> None:
|
|
labels = sorted(frame["y_true"].unique())
|
|
matrix = pd.crosstab(frame["y_true"], frame["y_pred"]).reindex(index=labels, columns=labels, fill_value=0)
|
|
matrix = matrix.div(matrix.sum(axis=1), axis=0) * 100
|
|
fig, ax = plt.subplots(figsize=(13.5, 11.5))
|
|
image = ax.imshow(matrix.to_numpy(), cmap=CMAP, vmin=0, vmax=35, aspect="auto")
|
|
ax.set_xticks(np.arange(len(labels)))
|
|
ax.set_yticks(np.arange(len(labels)))
|
|
ax.set_xticklabels(labels, rotation=90, fontsize=5.5)
|
|
ax.set_yticklabels(labels, fontsize=5.5)
|
|
ax.set_xlabel("Предсказанный персонаж", fontsize=12, color=DARK)
|
|
ax.set_ylabel("Истинный персонаж", fontsize=12, color=DARK)
|
|
ax.set_title("Полная матрица ошибок, % строки (раздаточный материал)", fontsize=16, color=DARK)
|
|
fig.colorbar(image, ax=ax, fraction=0.03, pad=0.015).set_label("% строки")
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "confusion_full34.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def pairs_chart(frame: pd.DataFrame) -> None:
|
|
pairs = top_confusions(frame, 18).sort_values()
|
|
fig, ax = plt.subplots(figsize=(12.6, 7.2))
|
|
colors = [RED if i >= len(pairs) - 5 else BROWN for i in range(len(pairs))]
|
|
ax.barh(pairs.index, pairs.values, color=colors, edgecolor="white", linewidth=0.6)
|
|
for index, value in enumerate(pairs.values):
|
|
ax.text(value + 0.6, index, str(value), va="center", fontsize=10, color=DARK)
|
|
ax.set_xlabel("Число ошибок за 3 повтора", fontsize=13, color=DARK)
|
|
ax.set_title("Самые частые пары «истинный → предсказанный»", fontsize=18, color=DARK)
|
|
ax.tick_params(axis="y", labelsize=10)
|
|
ax.grid(axis="x", alpha=0.2, color=DARK)
|
|
ax.set_xlim(0, pairs.max() * 1.16)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "confusion_pairs.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def per_class_chart(frame: pd.DataFrame) -> None:
|
|
first = frame[frame["repeat"] == frame["repeat"].min()]
|
|
labels = sorted(first["y_true"].unique())
|
|
rows = []
|
|
for label in labels:
|
|
mask = first["y_true"] == label
|
|
tp = int(((first["y_true"] == label) & (first["y_pred"] == label)).sum())
|
|
fp = int(((first["y_true"] != label) & (first["y_pred"] == label)).sum())
|
|
fn = int(((first["y_true"] == label) & (first["y_pred"] != label)).sum())
|
|
precision = tp / (tp + fp) if tp + fp else 0.0
|
|
recall = tp / (tp + fn) if tp + fn else 0.0
|
|
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
|
|
rows.append((label, f1, int(mask.sum())))
|
|
table = pd.DataFrame(rows, columns=["class", "f1", "support"]).sort_values("f1")
|
|
worst = table.head(12).sort_values("f1", ascending=False)
|
|
best = table.tail(12).sort_values("f1")
|
|
|
|
fig, axes = plt.subplots(1, 2, figsize=(13.2, 7.0), gridspec_kw={"width_ratios": [1, 1]})
|
|
axes[0].barh(worst["class"], worst["f1"], color=RED, edgecolor="white")
|
|
axes[0].set_title("Худшие 12 персонажей", fontsize=15, color=DARK)
|
|
axes[0].set_xlim(0, 0.72)
|
|
axes[0].grid(axis="x", alpha=0.2)
|
|
for index, (value, support) in enumerate(zip(worst["f1"], worst["support"])):
|
|
axes[0].text(value + 0.008, index, f"{value:.2f} (n={support})", va="center", fontsize=9, color=DARK)
|
|
axes[1].barh(best["class"], best["f1"], color=GREEN, edgecolor="white")
|
|
axes[1].set_title("Лучшие 12 персонажей", fontsize=15, color=DARK)
|
|
axes[1].set_xlim(0, 0.72)
|
|
axes[1].grid(axis="x", alpha=0.2)
|
|
for index, (value, support) in enumerate(zip(best["f1"], best["support"])):
|
|
axes[1].text(value + 0.008, index, f"{value:.2f} (n={support})", va="center", fontsize=9, color=DARK)
|
|
fig.suptitle("F-мера по персонажам: мягкое голосование, первый повтор", fontsize=17, color=DARK, y=0.99)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "per_class_f1.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def model_comparison() -> None:
|
|
table = pd.read_csv(METRICS)
|
|
wanted = [
|
|
("voting_et_char_lean", "Мягкое голосование"),
|
|
("et_char10000", "ExtraTrees, char=10000"),
|
|
("nested_feature_selection", "Вложенный выбор"),
|
|
("et_full", "ExtraTrees, полный"),
|
|
("logreg_lean_content300", "Logreg, lean + content"),
|
|
("stacking", "Стековое объединение"),
|
|
]
|
|
rows = []
|
|
for key, label in wanted:
|
|
row = table[table["label"] == key]
|
|
if not row.empty:
|
|
rows.append((label, float(row.iloc[0]["macro_f1_mean"]), float(row.iloc[0]["macro_f1_std"])))
|
|
rows.append(("Случайный уровень", 0.029, 0.0))
|
|
frame = pd.DataFrame(rows, columns=["label", "mean", "std"]).sort_values("mean")
|
|
fig, ax = plt.subplots(figsize=(12.4, 6.8))
|
|
colors = [GOLD if label == "Мягкое голосование" else (RED if label == "Случайный уровень" else GREEN) for label in frame["label"]]
|
|
ax.barh(frame["label"], frame["mean"], xerr=frame["std"], color=colors, edgecolor="white", capsize=4)
|
|
for index, (value, std) in enumerate(zip(frame["mean"], frame["std"])):
|
|
offset = value + max(std, 0.0) + 0.008
|
|
ax.text(offset, index, f"{value:.3f}", va="center", fontsize=11, color=DARK)
|
|
ax.set_xlim(0, 0.38)
|
|
ax.set_xlabel("Макро-усреднённая F-мера", fontsize=13, color=DARK)
|
|
ax.set_title("Сравнение моделей: среднее и стандартное отклонение", fontsize=18, color=DARK)
|
|
ax.grid(axis="x", alpha=0.2)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "model_comparison.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def feature_progression() -> None:
|
|
labels = [
|
|
"Только длина",
|
|
"Ручные признаки",
|
|
"Ручные + char/POS",
|
|
"+ content-LSA",
|
|
"ExtraTrees, char=10000",
|
|
"Мягкое голосование",
|
|
]
|
|
values = [0.035, 0.105, 0.182, 0.249, 0.276, 0.304]
|
|
fig, ax = plt.subplots(figsize=(12.4, 6.6))
|
|
colors = [BROWN, BROWN, GREEN, GREEN, GOLD, RED]
|
|
ax.bar(labels, values, color=colors, edgecolor="white")
|
|
for index, value in enumerate(values):
|
|
ax.text(index, value + 0.008, f"{value:.3f}", ha="center", fontsize=12, color=DARK)
|
|
ax.set_ylim(0, 0.35)
|
|
ax.set_ylabel("Макро-усреднённая F-мера", fontsize=13, color=DARK)
|
|
ax.set_title("Что добавляет качество", fontsize=18, color=DARK)
|
|
ax.tick_params(axis="x", rotation=18, labelsize=11)
|
|
ax.grid(axis="y", alpha=0.2)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "feature_progression.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def topic_control() -> None:
|
|
labels = ["Исходный\nExtraTrees", "Маскированный\nExtraTrees", "Исходный\nLogreg", "Маскированный\nLogreg"]
|
|
values = [0.2483, 0.1129, 0.1467, 0.1067]
|
|
fig, ax = plt.subplots(figsize=(10.8, 6.2))
|
|
colors = [GREEN, RED, GREEN, RED]
|
|
ax.bar(labels, values, color=colors, edgecolor="white")
|
|
for index, value in enumerate(values):
|
|
ax.text(index, value + 0.007, f"{value:.3f}", ha="center", fontsize=12, color=DARK)
|
|
ax.set_ylim(0, 0.30)
|
|
ax.set_ylabel("Макро-усреднённая F-мера", fontsize=13, color=DARK)
|
|
ax.set_title("Контроль содержания: замена содержательных слов на CONTENT", fontsize=17, color=DARK)
|
|
ax.tick_params(axis="x", labelsize=11)
|
|
ax.grid(axis="y", alpha=0.2)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "topic_control.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def hub_shares(frame: pd.DataFrame) -> None:
|
|
true = frame["y_true"].value_counts(normalize=True)
|
|
pred = frame["y_pred"].value_counts(normalize=True)
|
|
labels = pred.head(10).index.tolist()
|
|
x = np.arange(len(labels))
|
|
fig, ax = plt.subplots(figsize=(12.4, 6.2))
|
|
ax.bar(x - 0.2, [true.get(label, 0) * 100 for label in labels], width=0.4, label="Истинная доля", color=GREEN)
|
|
ax.bar(x + 0.2, [pred.get(label, 0) * 100 for label in labels], width=0.4, label="Доля предсказаний", color=RED)
|
|
ax.set_xticks(x)
|
|
ax.set_xticklabels(labels, rotation=25, ha="right", fontsize=11)
|
|
ax.set_ylabel("% реплик", fontsize=13, color=DARK)
|
|
ax.set_title("Классы-аттракторы: кого модель выбирает слишком часто", fontsize=17, color=DARK)
|
|
ax.legend(frameon=False)
|
|
ax.grid(axis="y", alpha=0.2)
|
|
fig.tight_layout()
|
|
fig.savefig(OUT / "hub_shares.png", dpi=220, bbox_inches="tight")
|
|
plt.close(fig)
|
|
|
|
|
|
def main() -> int:
|
|
frame = load_oof()
|
|
confusion_top12(frame)
|
|
confusion_full(frame)
|
|
pairs_chart(frame)
|
|
per_class_chart(frame)
|
|
model_comparison()
|
|
feature_progression()
|
|
topic_control()
|
|
hub_shares(frame)
|
|
print("Готово:", OUT)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|