Добавлены финальные эксперименты и итоговый отчёт

2026-10-26 13:27:53 +03:00
parent c3769b1d4a
commit 73c32d10c4
8 changed files with 1537 additions and 0 deletions
--- a/lab1/scripts/run_finalists.py
+++ b/lab1/scripts/run_finalists.py
@@ -0,0 +1,130 @@
+"""Сравнение финалистов и вариантов мягкого голосования."""
+
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import replace
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import VotingClassifier
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from stylometry.cv import run_repeated_cv
+from stylometry.datasets import load_feature_frame
+from stylometry.pipelines import FeatureConfig, build_pipeline, preset_config
+
+OUT = PROJECT_ROOT / "results" / "final" / "finalists"
+
+
+def lean(base: FeatureConfig, k: int) -> FeatureConfig:
+    groups = tuple(
+        group for group in base.groups
+        if group not in {"function_words", "pos", "vocabulary", "readability"}
+    )
+    return replace(base, groups=groups, lsa_variants=("content",), lsa_components=k)
+
+
+def save(result, name: str, config: dict) -> None:
+    directory = OUT / name
+    directory.mkdir(parents=True, exist_ok=True)
+    result.fold_metrics.to_csv(directory / "fold_metrics.csv", index=False, encoding="utf-8-sig")
+    result.oof.to_csv(directory / "oof_predictions.csv", index=False, encoding="utf-8-sig")
+    pooled, bootstrap = result.pooled_summary()
+    pooled.to_csv(directory / "pooled_metrics.csv", index=False, encoding="utf-8-sig")
+    with (directory / "summary.json").open("w", encoding="utf-8") as stream:
+        json.dump(
+            {
+                "config": config,
+                "fold_mean_macro_f1": float(result.fold_metrics["macro_f1"].mean()),
+                "fold_std_macro_f1": float(result.fold_metrics["macro_f1"].std(ddof=1)),
+                "pooled": pooled.to_dict(orient="records"),
+                "bootstrap": bootstrap,
+            },
+            stream,
+            ensure_ascii=False,
+            indent=2,
+            default=str,
+        )
+
+
+def main() -> int:
+    frame = load_feature_frame(verbose=True)
+    y = frame["character"].to_numpy()
+    base = preset_config("stylometry_pos_lsa")
+    configs = [
+        ("et_full", base, "extra_trees"),
+        ("et_char10000", replace(base, char_max_features=10000), "extra_trees"),
+        ("et_lean_content300", lean(base, 300), "extra_trees"),
+        ("et_lean_content400", lean(base, 400), "extra_trees"),
+        ("logreg_lean_content300", lean(base, 300), "logreg"),
+    ]
+    rows = []
+    for name, config, model in configs:
+        print(f"[finalists] {name}", flush=True)
+        result = run_repeated_cv(
+            build_pipeline(config, model), frame, y,
+            n_splits=5, n_repeats=5, seed=42, model_name=model, verbose=False,
+        )
+        save(result, name, {"name": name, "model": model})
+        pooled, bootstrap = result.pooled_summary()
+        first = pooled.iloc[0].to_dict()
+        rows.append({
+            "name": name,
+            "model": model,
+            "macro_f1": first.get("macro_f1"),
+            "accuracy": first.get("accuracy"),
+            "balanced_accuracy": first.get("balanced_accuracy"),
+            "top3_accuracy": first.get("top3_accuracy"),
+            "macro_f1_ci_lower": bootstrap["macro_f1"]["lower"],
+            "macro_f1_ci_upper": bootstrap["macro_f1"]["upper"],
+        })
+
+    voting_sets = {
+        "voting_et_char_lean": [
+            ("et_full", build_pipeline(base, "extra_trees")),
+            ("et_char10000", build_pipeline(replace(base, char_max_features=10000), "extra_trees")),
+            ("logreg_lean_content300", build_pipeline(lean(base, 300), "logreg")),
+        ],
+        "voting_four": [
+            ("et_full", build_pipeline(base, "extra_trees")),
+            ("rf_char10000", build_pipeline(replace(base, char_max_features=10000), "random_forest")),
+            ("et_char10000", build_pipeline(replace(base, char_max_features=10000), "extra_trees")),
+            ("logreg_lean_content300", build_pipeline(lean(base, 300), "logreg")),
+        ],
+    }
+    for name, estimators in voting_sets.items():
+        print(f"[finalists] {name}", flush=True)
+        voting = VotingClassifier(estimators=estimators, voting="soft", n_jobs=1)
+        result = run_repeated_cv(
+            voting, frame, y, n_splits=5, n_repeats=3, seed=42,
+            model_name=name, verbose=False,
+        )
+        save(result, name, {"name": name, "estimators": [key for key, _ in estimators]})
+        pooled, bootstrap = result.pooled_summary()
+        first = pooled.iloc[0].to_dict()
+        rows.append({
+            "name": name,
+            "model": "soft_voting",
+            "macro_f1": first.get("macro_f1"),
+            "accuracy": first.get("accuracy"),
+            "balanced_accuracy": first.get("balanced_accuracy"),
+            "top3_accuracy": first.get("top3_accuracy"),
+            "macro_f1_ci_lower": bootstrap["macro_f1"]["lower"],
+            "macro_f1_ci_upper": bootstrap["macro_f1"]["upper"],
+        })
+
+    table = pd.DataFrame(rows).sort_values("macro_f1", ascending=False)
+    OUT.mkdir(parents=True, exist_ok=True)
+    table.to_csv(OUT / "finalists.csv", index=False, encoding="utf-8-sig")
+    print(table.to_string(index=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())