144 lines
5.4 KiB
Python
144 lines
5.4 KiB
Python
"""Статистические критерии для сравнения моделей."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Sequence
|
||
|
||
import numpy as np
|
||
from scipy.stats import binomtest, friedmanchisquare, wilcoxon
|
||
from sklearn.metrics import accuracy_score, f1_score
|
||
|
||
|
||
def mcnemar_exact(
|
||
y_true: Sequence,
|
||
predictions_a: Sequence,
|
||
predictions_b: Sequence,
|
||
) -> dict[str, float]:
|
||
"""Точный критерий Мак-Немара для двух парных предсказаний."""
|
||
y_true = np.asarray(y_true)
|
||
correct_a = np.asarray(predictions_a) == y_true
|
||
correct_b = np.asarray(predictions_b) == y_true
|
||
b = int(np.sum(correct_a & ~correct_b))
|
||
c = int(np.sum(~correct_a & correct_b))
|
||
n = b + c
|
||
p_value = 1.0 if n == 0 else float(binomtest(min(b, c), n, 0.5).pvalue)
|
||
return {
|
||
"b": b,
|
||
"c": c,
|
||
"n_discordant": n,
|
||
"statistic": float(min(b, c)),
|
||
"p_value": p_value,
|
||
}
|
||
|
||
|
||
def wilcoxon_signed_rank(
|
||
scores_a: Sequence[float],
|
||
scores_b: Sequence[float],
|
||
) -> dict[str, float]:
|
||
"""Парный критерий Уилкоксона для результатов по фолдам."""
|
||
scores_a = np.asarray(scores_a, dtype=float)
|
||
scores_b = np.asarray(scores_b, dtype=float)
|
||
differences = scores_a - scores_b
|
||
if np.allclose(differences, 0):
|
||
return {"statistic": 0.0, "p_value": 1.0, "n": int(len(scores_a))}
|
||
statistic, p_value = wilcoxon(scores_a, scores_b, alternative="two-sided")
|
||
return {
|
||
"statistic": float(statistic),
|
||
"p_value": float(p_value),
|
||
"n": int(len(scores_a)),
|
||
}
|
||
|
||
|
||
def friedman_test(*score_arrays: Sequence[float]) -> dict[str, float]:
|
||
"""Критерий Фридмана для сравнения нескольких моделей."""
|
||
arrays = [np.asarray(values, dtype=float) for values in score_arrays]
|
||
if len(arrays) < 3:
|
||
raise ValueError("Для критерия Фридмана нужно хотя бы три модели.")
|
||
if len({len(values) for values in arrays}) != 1:
|
||
raise ValueError("Все массивы должны быть одинаковой длины.")
|
||
statistic, p_value = friedmanchisquare(*arrays)
|
||
return {"statistic": float(statistic), "p_value": float(p_value)}
|
||
|
||
|
||
def holm_bonferroni(p_values: Sequence[float]) -> list[float]:
|
||
"""Поправка Холма—Бонферрони для множественных сравнений."""
|
||
values = np.asarray(p_values, dtype=float)
|
||
n = len(values)
|
||
order = np.argsort(values)
|
||
adjusted = np.empty(n, dtype=float)
|
||
previous = 0.0
|
||
for rank, index in enumerate(order):
|
||
value = min(1.0, values[index] * (n - rank))
|
||
previous = max(previous, value)
|
||
adjusted[index] = previous
|
||
return adjusted.tolist()
|
||
|
||
|
||
def bootstrap_difference(
|
||
y_true: Sequence,
|
||
predictions_a: Sequence,
|
||
predictions_b: Sequence,
|
||
metric: str = "macro_f1",
|
||
labels: Sequence | None = None,
|
||
n_boot: int = 1000,
|
||
seed: int = 42,
|
||
) -> dict[str, float]:
|
||
"""Доверительный интервал разницы метрик двух моделей."""
|
||
y_true = np.asarray(y_true)
|
||
predictions_a = np.asarray(predictions_a)
|
||
predictions_b = np.asarray(predictions_b)
|
||
labels_array = np.asarray(labels) if labels is not None else np.unique(y_true)
|
||
|
||
def _metric(true_values: np.ndarray, predicted_values: np.ndarray) -> float:
|
||
if metric == "macro_f1":
|
||
return float(
|
||
f1_score(
|
||
true_values,
|
||
predicted_values,
|
||
labels=labels_array,
|
||
average="macro",
|
||
zero_division=0,
|
||
)
|
||
)
|
||
if metric == "accuracy":
|
||
return float(accuracy_score(true_values, predicted_values))
|
||
raise ValueError(f"Неизвестная метрика: {metric}")
|
||
|
||
rng = np.random.default_rng(seed)
|
||
differences = []
|
||
n_samples = len(y_true)
|
||
for _ in range(n_boot):
|
||
indices = rng.integers(0, n_samples, n_samples)
|
||
differences.append(
|
||
_metric(y_true[indices], predictions_a[indices])
|
||
- _metric(y_true[indices], predictions_b[indices])
|
||
)
|
||
values = np.asarray(differences)
|
||
return {
|
||
"mean_difference": float(values.mean()),
|
||
"lower": float(np.percentile(values, 2.5)),
|
||
"upper": float(np.percentile(values, 97.5)),
|
||
"n_boot": int(n_boot),
|
||
}
|
||
|
||
|
||
def cochran_q(correctness_matrix: np.ndarray) -> dict[str, float]:
|
||
"""Критерий Кокрана для сравнения нескольких бинарных классификаторов.
|
||
|
||
``correctness_matrix`` имеет форму (объекты, модели), значения 0/1.
|
||
"""
|
||
matrix = np.asarray(correctness_matrix, dtype=float)
|
||
if matrix.ndim != 2:
|
||
raise ValueError("Ожидается матрица (объекты, модели).")
|
||
k = matrix.shape[1]
|
||
column_sums = matrix.sum(axis=0)
|
||
row_sums = matrix.sum(axis=1)
|
||
total = matrix.sum()
|
||
numerator = (k - 1) * (k * np.sum(column_sums**2) - total**2)
|
||
denominator = k * total - np.sum(row_sums**2)
|
||
statistic = 0.0 if denominator == 0 else float(numerator / denominator)
|
||
from scipy.stats import chi2
|
||
|
||
p_value = float(chi2.sf(statistic, df=k - 1))
|
||
return {"statistic": statistic, "p_value": p_value, "df": k - 1}
|