131 lines
5.1 KiB
Python
131 lines
5.1 KiB
Python
"""Сравнение финалистов и вариантов мягкого голосования."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from sklearn.ensemble import VotingClassifier
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from stylometry.cv import run_repeated_cv
|
|
from stylometry.datasets import load_feature_frame
|
|
from stylometry.pipelines import FeatureConfig, build_pipeline, preset_config
|
|
|
|
OUT = PROJECT_ROOT / "results" / "final" / "finalists"
|
|
|
|
|
|
def lean(base: FeatureConfig, k: int) -> FeatureConfig:
|
|
groups = tuple(
|
|
group for group in base.groups
|
|
if group not in {"function_words", "pos", "vocabulary", "readability"}
|
|
)
|
|
return replace(base, groups=groups, lsa_variants=("content",), lsa_components=k)
|
|
|
|
|
|
def save(result, name: str, config: dict) -> None:
|
|
directory = OUT / name
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
result.fold_metrics.to_csv(directory / "fold_metrics.csv", index=False, encoding="utf-8-sig")
|
|
result.oof.to_csv(directory / "oof_predictions.csv", index=False, encoding="utf-8-sig")
|
|
pooled, bootstrap = result.pooled_summary()
|
|
pooled.to_csv(directory / "pooled_metrics.csv", index=False, encoding="utf-8-sig")
|
|
with (directory / "summary.json").open("w", encoding="utf-8") as stream:
|
|
json.dump(
|
|
{
|
|
"config": config,
|
|
"fold_mean_macro_f1": float(result.fold_metrics["macro_f1"].mean()),
|
|
"fold_std_macro_f1": float(result.fold_metrics["macro_f1"].std(ddof=1)),
|
|
"pooled": pooled.to_dict(orient="records"),
|
|
"bootstrap": bootstrap,
|
|
},
|
|
stream,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
default=str,
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
frame = load_feature_frame(verbose=True)
|
|
y = frame["character"].to_numpy()
|
|
base = preset_config("stylometry_pos_lsa")
|
|
configs = [
|
|
("et_full", base, "extra_trees"),
|
|
("et_char10000", replace(base, char_max_features=10000), "extra_trees"),
|
|
("et_lean_content300", lean(base, 300), "extra_trees"),
|
|
("et_lean_content400", lean(base, 400), "extra_trees"),
|
|
("logreg_lean_content300", lean(base, 300), "logreg"),
|
|
]
|
|
rows = []
|
|
for name, config, model in configs:
|
|
print(f"[finalists] {name}", flush=True)
|
|
result = run_repeated_cv(
|
|
build_pipeline(config, model), frame, y,
|
|
n_splits=5, n_repeats=5, seed=42, model_name=model, verbose=False,
|
|
)
|
|
save(result, name, {"name": name, "model": model})
|
|
pooled, bootstrap = result.pooled_summary()
|
|
first = pooled.iloc[0].to_dict()
|
|
rows.append({
|
|
"name": name,
|
|
"model": model,
|
|
"macro_f1": first.get("macro_f1"),
|
|
"accuracy": first.get("accuracy"),
|
|
"balanced_accuracy": first.get("balanced_accuracy"),
|
|
"top3_accuracy": first.get("top3_accuracy"),
|
|
"macro_f1_ci_lower": bootstrap["macro_f1"]["lower"],
|
|
"macro_f1_ci_upper": bootstrap["macro_f1"]["upper"],
|
|
})
|
|
|
|
voting_sets = {
|
|
"voting_et_char_lean": [
|
|
("et_full", build_pipeline(base, "extra_trees")),
|
|
("et_char10000", build_pipeline(replace(base, char_max_features=10000), "extra_trees")),
|
|
("logreg_lean_content300", build_pipeline(lean(base, 300), "logreg")),
|
|
],
|
|
"voting_four": [
|
|
("et_full", build_pipeline(base, "extra_trees")),
|
|
("rf_char10000", build_pipeline(replace(base, char_max_features=10000), "random_forest")),
|
|
("et_char10000", build_pipeline(replace(base, char_max_features=10000), "extra_trees")),
|
|
("logreg_lean_content300", build_pipeline(lean(base, 300), "logreg")),
|
|
],
|
|
}
|
|
for name, estimators in voting_sets.items():
|
|
print(f"[finalists] {name}", flush=True)
|
|
voting = VotingClassifier(estimators=estimators, voting="soft", n_jobs=1)
|
|
result = run_repeated_cv(
|
|
voting, frame, y, n_splits=5, n_repeats=3, seed=42,
|
|
model_name=name, verbose=False,
|
|
)
|
|
save(result, name, {"name": name, "estimators": [key for key, _ in estimators]})
|
|
pooled, bootstrap = result.pooled_summary()
|
|
first = pooled.iloc[0].to_dict()
|
|
rows.append({
|
|
"name": name,
|
|
"model": "soft_voting",
|
|
"macro_f1": first.get("macro_f1"),
|
|
"accuracy": first.get("accuracy"),
|
|
"balanced_accuracy": first.get("balanced_accuracy"),
|
|
"top3_accuracy": first.get("top3_accuracy"),
|
|
"macro_f1_ci_lower": bootstrap["macro_f1"]["lower"],
|
|
"macro_f1_ci_upper": bootstrap["macro_f1"]["upper"],
|
|
})
|
|
|
|
table = pd.DataFrame(rows).sort_values("macro_f1", ascending=False)
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
table.to_csv(OUT / "finalists.csv", index=False, encoding="utf-8-sig")
|
|
print(table.to_string(index=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|