Первичная версия: стилометрия реплик, LSA и протокол экспериментов
This commit is contained in:
293
lab1/stylometry/cv.py
Normal file
293
lab1/stylometry/cv.py
Normal file
@@ -0,0 +1,293 @@
|
||||
"""Схемы кросс-валидации и сбор out-of-fold предсказаний."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.base import clone
|
||||
from sklearn.model_selection import StratifiedKFold
|
||||
|
||||
from .evaluation import (
|
||||
align_scores,
|
||||
classification_metrics,
|
||||
metrics_summary,
|
||||
summarize_pooled_predictions,
|
||||
)
|
||||
from .pipelines import make_grid_search
|
||||
|
||||
|
||||
@dataclass
|
||||
class CVResult:
|
||||
"""Результат повторной стратифицированной кросс-валидации."""
|
||||
|
||||
fold_metrics: pd.DataFrame
|
||||
oof: pd.DataFrame
|
||||
labels: np.ndarray
|
||||
scores_by_repeat: dict[int, np.ndarray] = field(default_factory=dict)
|
||||
n_splits: int = 5
|
||||
n_repeats: int = 1
|
||||
model_name: str = ""
|
||||
|
||||
def summary(self) -> pd.DataFrame:
|
||||
return metrics_summary(self.fold_metrics)
|
||||
|
||||
def pooled_summary(self) -> tuple[pd.DataFrame, dict[str, float]]:
|
||||
return summarize_pooled_predictions(
|
||||
self.oof, self.labels, self.scores_by_repeat
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NestedCVResult(CVResult):
|
||||
"""Результат вложенной кросс-валидации с настройкой гиперпараметров."""
|
||||
|
||||
best_params: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _predict_scores(pipeline: Any, X: pd.DataFrame) -> np.ndarray | None:
|
||||
if hasattr(pipeline, "predict_proba"):
|
||||
try:
|
||||
return np.asarray(pipeline.predict_proba(X), dtype=float)
|
||||
except Exception:
|
||||
pass
|
||||
if hasattr(pipeline, "decision_function"):
|
||||
try:
|
||||
scores = np.asarray(pipeline.decision_function(X), dtype=float)
|
||||
if scores.ndim == 1:
|
||||
scores = np.column_stack([-scores, scores])
|
||||
return scores
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _model_classes(pipeline: Any) -> np.ndarray:
|
||||
if hasattr(pipeline, "named_steps") and "model" in pipeline.named_steps:
|
||||
return np.asarray(pipeline.named_steps["model"].classes_)
|
||||
return np.asarray(pipeline.classes_)
|
||||
|
||||
|
||||
def validate_folds(
|
||||
y: Sequence,
|
||||
splits: Iterable[tuple[np.ndarray, np.ndarray]],
|
||||
labels: Sequence,
|
||||
) -> None:
|
||||
"""Проверяет, что каждый класс есть и в обучении, и в тесте."""
|
||||
y = np.asarray(y)
|
||||
labels_set = set(labels)
|
||||
for fold_index, (train_idx, test_idx) in enumerate(splits):
|
||||
train_labels = set(y[train_idx])
|
||||
test_labels = set(y[test_idx])
|
||||
missing_train = labels_set - train_labels
|
||||
missing_test = labels_set - test_labels
|
||||
if missing_train or missing_test:
|
||||
raise ValueError(
|
||||
f"Фолд {fold_index}: в обучении нет классов {sorted(missing_train)}, "
|
||||
f"в тесте нет классов {sorted(missing_test)}. "
|
||||
"Увеличьте размер выборки или уменьшите число фолдов."
|
||||
)
|
||||
|
||||
|
||||
def _line_ids(X: pd.DataFrame, indices: np.ndarray) -> np.ndarray:
|
||||
if "line_id" in X.columns:
|
||||
return X.iloc[indices]["line_id"].to_numpy()
|
||||
return np.asarray(indices)
|
||||
|
||||
|
||||
def _order_scores_like_oof(X: pd.DataFrame, rows: list[dict[str, Any]]) -> np.ndarray:
|
||||
"""Возвращает порядок строк матрицы score-ов, соответствующий OOF-таблице."""
|
||||
if "line_id" in X.columns:
|
||||
positions = {
|
||||
int(value): index for index, value in enumerate(X["line_id"].to_numpy())
|
||||
}
|
||||
return np.asarray([positions[int(row["line_id"])] for row in rows], dtype=int)
|
||||
return np.asarray([int(row["line_id"]) for row in rows], dtype=int)
|
||||
|
||||
|
||||
def run_repeated_cv(
|
||||
pipeline: Any,
|
||||
X: pd.DataFrame,
|
||||
y: Sequence,
|
||||
n_splits: int = 5,
|
||||
n_repeats: int = 5,
|
||||
seed: int = 42,
|
||||
model_name: str = "",
|
||||
verbose: bool = True,
|
||||
) -> CVResult:
|
||||
"""Повторная стратифицированная кросс-валидация.
|
||||
|
||||
Все преобразования внутри ``pipeline`` обучаются только на обучающем
|
||||
блоке. Out-of-fold предсказания собираются для каждого повтора.
|
||||
"""
|
||||
y = np.asarray(y)
|
||||
labels = np.unique(y)
|
||||
n_samples = len(y)
|
||||
fold_rows: list[dict[str, Any]] = []
|
||||
oof_rows: list[dict[str, Any]] = []
|
||||
scores_by_repeat: dict[int, np.ndarray] = {}
|
||||
|
||||
for repeat in range(n_repeats):
|
||||
splitter = StratifiedKFold(
|
||||
n_splits=n_splits, shuffle=True, random_state=seed + repeat
|
||||
)
|
||||
splits = list(splitter.split(X, y))
|
||||
validate_folds(y, splits, labels)
|
||||
scores_oof = np.full((n_samples, len(labels)), np.nan, dtype=float)
|
||||
oof_start = len(oof_rows)
|
||||
|
||||
for fold, (train_idx, test_idx) in enumerate(splits, start=1):
|
||||
model = clone(pipeline)
|
||||
model.fit(X.iloc[train_idx], y[train_idx])
|
||||
y_pred = model.predict(X.iloc[test_idx])
|
||||
raw_scores = _predict_scores(model, X.iloc[test_idx])
|
||||
if raw_scores is not None:
|
||||
scores = align_scores(
|
||||
raw_scores, _model_classes(model), labels, fill_value=0.0
|
||||
)
|
||||
scores_oof[test_idx] = scores
|
||||
else:
|
||||
scores = None
|
||||
|
||||
metrics = classification_metrics(
|
||||
y[test_idx], y_pred, y_score=scores, labels=labels
|
||||
)
|
||||
metrics.update(
|
||||
{
|
||||
"repeat": repeat,
|
||||
"fold": fold,
|
||||
"n_train": int(len(train_idx)),
|
||||
"n_test": int(len(test_idx)),
|
||||
}
|
||||
)
|
||||
fold_rows.append(metrics)
|
||||
|
||||
for row_index, sample_index in enumerate(test_idx):
|
||||
oof_rows.append(
|
||||
{
|
||||
"repeat": repeat,
|
||||
"fold": fold,
|
||||
"line_id": int(_line_ids(X, np.asarray([sample_index]))[0]),
|
||||
"character": str(X.iloc[sample_index].get("character", "")),
|
||||
"y_true": y[sample_index],
|
||||
"y_pred": y_pred[row_index],
|
||||
}
|
||||
)
|
||||
if verbose:
|
||||
print(
|
||||
f"[cv] повтор {repeat + 1}/{n_repeats}, "
|
||||
f"фолд {fold}/{n_splits}: macro-F1={metrics['macro_f1']:.4f}"
|
||||
)
|
||||
|
||||
if not np.isnan(scores_oof).any():
|
||||
repeat_rows = sorted(oof_rows[oof_start:], key=lambda row: int(row["line_id"]))
|
||||
order = _order_scores_like_oof(X, repeat_rows)
|
||||
scores_by_repeat[repeat] = scores_oof[order]
|
||||
|
||||
oof = pd.DataFrame(oof_rows)
|
||||
if not oof.empty:
|
||||
oof = oof.sort_values(["repeat", "line_id"]).reset_index(drop=True)
|
||||
return CVResult(
|
||||
fold_metrics=pd.DataFrame(fold_rows),
|
||||
oof=oof,
|
||||
labels=labels,
|
||||
scores_by_repeat=scores_by_repeat,
|
||||
n_splits=n_splits,
|
||||
n_repeats=n_repeats,
|
||||
model_name=model_name,
|
||||
)
|
||||
|
||||
|
||||
def run_nested_cv(
|
||||
pipeline: Any,
|
||||
X: pd.DataFrame,
|
||||
y: Sequence,
|
||||
model_name: str,
|
||||
outer_splits: int = 5,
|
||||
inner_splits: int = 3,
|
||||
seed: int = 42,
|
||||
verbose: bool = True,
|
||||
) -> NestedCVResult:
|
||||
"""Вложенная кросс-валидация: настройка гиперпараметров внутри outer-фолдов."""
|
||||
y = np.asarray(y)
|
||||
labels = np.unique(y)
|
||||
n_samples = len(y)
|
||||
outer = StratifiedKFold(
|
||||
n_splits=outer_splits, shuffle=True, random_state=seed
|
||||
)
|
||||
splits = list(outer.split(X, y))
|
||||
validate_folds(y, splits, labels)
|
||||
|
||||
fold_rows: list[dict[str, Any]] = []
|
||||
oof_rows: list[dict[str, Any]] = []
|
||||
best_params: list[dict[str, Any]] = []
|
||||
scores_oof = np.full((n_samples, len(labels)), np.nan, dtype=float)
|
||||
|
||||
for fold, (train_idx, test_idx) in enumerate(splits, start=1):
|
||||
search = make_grid_search(
|
||||
pipeline, model_name, cv=inner_splits, scoring="f1_macro", n_jobs=-1
|
||||
)
|
||||
search.fit(X.iloc[train_idx], y[train_idx])
|
||||
model = search.best_estimator_
|
||||
y_pred = model.predict(X.iloc[test_idx])
|
||||
raw_scores = _predict_scores(model, X.iloc[test_idx])
|
||||
if raw_scores is not None:
|
||||
scores = align_scores(
|
||||
raw_scores, _model_classes(model), labels, fill_value=0.0
|
||||
)
|
||||
scores_oof[test_idx] = scores
|
||||
else:
|
||||
scores = None
|
||||
|
||||
metrics = classification_metrics(
|
||||
y[test_idx], y_pred, y_score=scores, labels=labels
|
||||
)
|
||||
metrics.update(
|
||||
{
|
||||
"repeat": 0,
|
||||
"fold": fold,
|
||||
"n_train": int(len(train_idx)),
|
||||
"n_test": int(len(test_idx)),
|
||||
}
|
||||
)
|
||||
fold_rows.append(metrics)
|
||||
best_params.append(dict(search.best_params_))
|
||||
for row_index, sample_index in enumerate(test_idx):
|
||||
oof_rows.append(
|
||||
{
|
||||
"repeat": 0,
|
||||
"fold": fold,
|
||||
"line_id": int(_line_ids(X, np.asarray([sample_index]))[0]),
|
||||
"character": str(X.iloc[sample_index].get("character", "")),
|
||||
"y_true": y[sample_index],
|
||||
"y_pred": y_pred[row_index],
|
||||
}
|
||||
)
|
||||
if verbose:
|
||||
print(
|
||||
f"[nested-cv] фолд {fold}/{outer_splits}: "
|
||||
f"macro-F1={metrics['macro_f1']:.4f}, "
|
||||
f"best={search.best_params_}"
|
||||
)
|
||||
|
||||
scores_by_repeat = {}
|
||||
if not np.isnan(scores_oof).any():
|
||||
ordered_rows = sorted(oof_rows, key=lambda row: int(row["line_id"]))
|
||||
order = _order_scores_like_oof(X, ordered_rows)
|
||||
scores_by_repeat[0] = scores_oof[order]
|
||||
|
||||
oof = pd.DataFrame(oof_rows)
|
||||
if not oof.empty:
|
||||
oof = oof.sort_values(["repeat", "line_id"]).reset_index(drop=True)
|
||||
return NestedCVResult(
|
||||
fold_metrics=pd.DataFrame(fold_rows),
|
||||
oof=oof,
|
||||
labels=labels,
|
||||
scores_by_repeat=scores_by_repeat,
|
||||
n_splits=outer_splits,
|
||||
n_repeats=1,
|
||||
model_name=model_name,
|
||||
best_params=best_params,
|
||||
)
|
||||
Reference in New Issue
Block a user