Mermutef/NLP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

opencode tests and his result

Browse Source
This commit is contained in:
folen
2026-10-26 05:44:25 +03:00
parent 2c6bf4653a
commit ac7c6f0b92
12 changed files with 3032 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

336
lab1/scripts/analyze_followup.py Executable file
View File

@@ -0,0 +1,336 @@
#!/usr/bin/env python
"""Сводка follow-up экспериментов и парные сравнения по повторам.
Скрипт читает каталоги results/followup/<label>/, созданные
scripts/run_followup_experiments.sh, и формирует:
- summary.csv / summary.md — среднее macro-F1 по повторам, а не только repeat 0;
- comparisons.csv / comparisons.md — парные сравнения с McNemar и bootstrap;
- comparisons/<A>__vs__<B>.json — детали по каждому повтору.
Пример:
python scripts/analyze_followup.py \
--followup-dir results/followup \
--pairs final_rf:ref_logreg final_rf:ref_no_lsa
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
import numpy as np
import pandas as pd
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT))
from stylometry.evaluation import classification_metrics # noqa: E402
from stylometry.stats import bootstrap_difference, mcnemar_exact # noqa: E402
def _json_load(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def _read_run(run_dir: Path) -> tuple[dict[str, Any], pd.DataFrame]:
metadata = _json_load(run_dir / "run_metadata.json")
pooled_path = run_dir / "pooled_metrics.csv"
if pooled_path.exists():
pooled = pd.read_csv(pooled_path)
else:
pooled = pd.DataFrame([metadata.get("metrics", {})])
return metadata, pooled
def _feature_summary(config: dict[str, Any]) -> str:
keys = (
"groups",
"lsa_variants",
"lsa_components",
"min_df",
"char_max_features",
"pos_max_features",
"char_ngram_range",
"pos_ngram_range",
"block_weights",
)
features = config.get("feature_config", {})
return json.dumps({key: features.get(key) for key in keys if key in features}, ensure_ascii=False)
def build_summary(followup_dir: Path) -> pd.DataFrame:
rows: list[dict[str, Any]] = []
for run_dir in sorted(followup_dir.iterdir()):
if not run_dir.is_dir() or not (run_dir / "run_metadata.json").exists():
continue
metadata, pooled = _read_run(run_dir)
config = metadata.get("config", {})
bootstrap = metadata.get("bootstrap", {}).get("macro_f1", {})
macro = pd.to_numeric(pooled.get("macro_f1"), errors="coerce")
accuracy = pd.to_numeric(pooled.get("accuracy"), errors="coerce")
balanced = pd.to_numeric(pooled.get("balanced_accuracy"), errors="coerce")
top3 = pd.to_numeric(pooled.get("top3_accuracy"), errors="coerce")
command_path = run_dir / "command.txt"
rows.append(
{
"label": run_dir.name,
"preset": config.get("preset"),
"model": config.get("model"),
"folds": config.get("folds"),
"n_repeats": int(len(pooled)),
"n_samples": config.get("n_samples"),
"mean_macro_f1": float(macro.mean()),
"std_macro_f1": float(macro.std(ddof=1)) if len(macro) > 1 else 0.0,
"min_macro_f1": float(macro.min()),
"max_macro_f1": float(macro.max()),
"repeat0_macro_f1": float(macro.iloc[0]),
"mean_accuracy": float(accuracy.mean()),
"mean_balanced_accuracy": float(balanced.mean()),
"mean_top3_accuracy": float(top3.mean()),
"ci_lower_repeat0": bootstrap.get("lower"),
"ci_upper_repeat0": bootstrap.get("upper"),
"model_params": json.dumps(config.get("model_params", {}), ensure_ascii=False),
"feature_config": _feature_summary(config),
"command": command_path.read_text(encoding="utf-8").strip()
if command_path.exists()
else "",
}
)
summary = pd.DataFrame(rows)
if summary.empty:
return summary
reference = summary.loc[summary["label"] == "ref_logreg", "mean_macro_f1"]
if not reference.empty:
summary["delta_vs_ref_logreg"] = summary["mean_macro_f1"] - float(reference.iloc[0])
else:
summary["delta_vs_ref_logreg"] = np.nan
summary = summary.sort_values("mean_macro_f1", ascending=False).reset_index(drop=True)
summary.insert(0, "rank", np.arange(1, len(summary) + 1))
return summary
def _compare_pair(
followup_dir: Path,
label_a: str,
label_b: str,
n_boot: int,
seed: int,
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
dir_a = followup_dir / label_a
dir_b = followup_dir / label_b
if not (dir_a / "oof_predictions.csv").exists() or not (dir_b / "oof_predictions.csv").exists():
print(f"[compare] пропуск {label_a} vs {label_b}: нет oof_predictions.csv")
return None, []
oof_a = pd.read_csv(dir_a / "oof_predictions.csv")
oof_b = pd.read_csv(dir_b / "oof_predictions.csv")
common_repeats = sorted(set(oof_a["repeat"].unique()) & set(oof_b["repeat"].unique()))
if not common_repeats:
print(f"[compare] пропуск {label_a} vs {label_b}: нет общих повторов")
return None, []
details: list[dict[str, Any]] = []
for repeat in common_repeats:
first = oof_a.loc[oof_a["repeat"] == repeat, ["line_id", "y_true", "y_pred"]]
second = oof_b.loc[oof_b["repeat"] == repeat, ["line_id", "y_true", "y_pred"]]
merged = first.merge(
second,
on=["line_id", "y_true"],
suffixes=("_a", "_b"),
how="inner",
)
if merged.empty:
continue
labels = sorted(merged["y_true"].unique())
metrics_a = classification_metrics(merged["y_true"], merged["y_pred_a"], labels=labels)
metrics_b = classification_metrics(merged["y_true"], merged["y_pred_b"], labels=labels)
mcnemar = mcnemar_exact(merged["y_true"], merged["y_pred_a"], merged["y_pred_b"])
difference = bootstrap_difference(
merged["y_true"],
merged["y_pred_a"],
merged["y_pred_b"],
metric="macro_f1",
labels=labels,
n_boot=n_boot,
seed=seed + int(repeat),
)
details.append(
{
"repeat": int(repeat),
"n_samples": int(len(merged)),
"macro_f1_a": metrics_a["macro_f1"],
"macro_f1_b": metrics_b["macro_f1"],
"macro_f1_diff": metrics_a["macro_f1"] - metrics_b["macro_f1"],
"accuracy_a": metrics_a["accuracy"],
"accuracy_b": metrics_b["accuracy"],
"accuracy_diff": metrics_a["accuracy"] - metrics_b["accuracy"],
"mcnemar_p": mcnemar["p_value"],
"mcnemar_b": mcnemar["b"],
"mcnemar_c": mcnemar["c"],
"bootstrap_mean_diff": difference["mean_difference"],
"bootstrap_lower": difference["lower"],
"bootstrap_upper": difference["upper"],
}
)
if not details:
print(f"[compare] пропуск {label_a} vs {label_b}: не удалось сопоставить строки")
return None, []
diffs = np.asarray([row["macro_f1_diff"] for row in details], dtype=float)
p_values = np.asarray([row["mcnemar_p"] for row in details], dtype=float)
ci_lower = np.asarray([row["bootstrap_lower"] for row in details], dtype=float)
ci_upper = np.asarray([row["bootstrap_upper"] for row in details], dtype=float)
summary = {
"pair": f"{label_a} vs {label_b}",
"label_a": label_a,
"label_b": label_b,
"n_repeats": len(details),
"mean_macro_f1_a": float(np.mean([row["macro_f1_a"] for row in details])),
"mean_macro_f1_b": float(np.mean([row["macro_f1_b"] for row in details])),
"mean_diff": float(diffs.mean()),
"std_diff": float(diffs.std(ddof=1)) if len(diffs) > 1 else 0.0,
"min_diff": float(diffs.min()),
"max_diff": float(diffs.max()),
"n_positive": int((diffs > 0).sum()),
"n_negative": int((diffs < 0).sum()),
"mean_mcnemar_p": float(p_values.mean()),
"min_mcnemar_p": float(p_values.min()),
"mean_ci_lower": float(ci_lower.mean()),
"mean_ci_upper": float(ci_upper.mean()),
"n_ci_excludes_zero": int(((ci_lower > 0) | (ci_upper < 0)).sum()),
}
return summary, details
def _format_markdown(summary: pd.DataFrame) -> str:
columns = [
"rank",
"label",
"model",
"preset",
"n_repeats",
"mean_macro_f1",
"std_macro_f1",
"mean_accuracy",
"mean_top3_accuracy",
"delta_vs_ref_logreg",
]
table = summary[columns].copy()
return table.to_markdown(index=False, floatfmt=".4f")
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--followup-dir",
default=str(PROJECT_ROOT / "results" / "followup"),
help="Каталог с подкаталогами запусков.",
)
parser.add_argument(
"--pairs",
nargs="*",
default=[],
help="Пары для сравнения в формате label_a:label_b.",
)
parser.add_argument("--n-boot", type=int, default=1000)
parser.add_argument("--seed", type=int, default=42)
args = parser.parse_args(argv)
followup_dir = Path(args.followup_dir)
if not followup_dir.exists():
raise FileNotFoundError(f"Нет каталога: {followup_dir}")
summary = build_summary(followup_dir)
if summary.empty:
print(f"[summary] в {followup_dir} нет запусков с run_metadata.json")
return 1
summary_path = followup_dir / "summary.csv"
summary.to_csv(summary_path, index=False, encoding="utf-8-sig")
markdown = [
"# Follow-up: сводка запусков",
"",
"Среднее и стандартное отклонение посчитаны по pooled-метрикам каждого",
"повтора (`pooled_metrics.csv`), а не только по repeat 0.",
"",
_format_markdown(summary),
"",
]
(followup_dir / "summary.md").write_text("\n".join(markdown), encoding="utf-8")
print(f"[summary] {summary_path}")
print(
summary[
["rank", "label", "model", "n_repeats", "mean_macro_f1", "std_macro_f1"]
].to_string(index=False)
)
comparison_rows: list[dict[str, Any]] = []
comparisons_dir = followup_dir / "comparisons"
comparisons_dir.mkdir(exist_ok=True)
for pair in args.pairs:
if ":" not in pair:
print(f"[compare] некорректная пара: {pair}")
continue
label_a, label_b = pair.split(":", 1)
summary_row, details = _compare_pair(
followup_dir,
label_a.strip(),
label_b.strip(),
n_boot=args.n_boot,
seed=args.seed,
)
if summary_row is None:
continue
comparison_rows.append(summary_row)
payload = {"summary": summary_row, "details": details}
output = comparisons_dir / f"{label_a.strip()}__vs__{label_b.strip()}.json"
output.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
print(
f"[compare] {summary_row['pair']}: "
f"diff={summary_row['mean_diff']:+.4f}, "
f"{summary_row['n_positive']}/{summary_row['n_repeats']} повторов в плюс, "
f"mean McNemar p={summary_row['mean_mcnemar_p']:.4f}"
)
if comparison_rows:
comparisons = pd.DataFrame(comparison_rows).sort_values("mean_diff", ascending=False)
comparisons.to_csv(followup_dir / "comparisons.csv", index=False, encoding="utf-8-sig")
columns = [
"pair",
"n_repeats",
"mean_macro_f1_a",
"mean_macro_f1_b",
"mean_diff",
"std_diff",
"n_positive",
"mean_mcnemar_p",
"mean_ci_lower",
"mean_ci_upper",
"n_ci_excludes_zero",
]
comparison_markdown = [
"# Follow-up: парные сравнения",
"",
"`mean_diff` — средняя разница macro-F1 (A − B) по общим повторам.",
"`n_positive` — сколько повторов дали положительную разницу.",
"`n_ci_excludes_zero` — сколько повторов имеют bootstrap-интервал,",
"не содержащий ноль (описательно, без поправки на множественность).",
"",
comparisons[columns].to_markdown(index=False, floatfmt=".4f"),
"",
]
(followup_dir / "comparisons.md").write_text(
"\n".join(comparison_markdown), encoding="utf-8"
)
print(f"[compare] {followup_dir / 'comparisons.csv'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

824
lab1/scripts/run_followup_experiments.sh Executable file
View File

@@ -0,0 +1,824 @@
#!/usr/bin/env bash
# Follow-up серия экспериментов по итогам предварительных прогонов.
#
# Запуск из каталога lab1:
# bash scripts/run_followup_experiments.sh
#
# С другим интерпретатором:
# PYTHON=../.venv/bin/python bash scripts/run_followup_experiments.sh
#
# Быстрая проверка (1 повтор, без сетки и seed-робастности):
# QUICK=1 bash scripts/run_followup_experiments.sh
#
# Только воспроизведение и финальная модель, без тяжёлых стадий:
# RUN_GRID=0 RUN_SEEDS=0 bash scripts/run_followup_experiments.sh
#
# Все результаты складываются в results/followup/<label>/, команды и
# сгенерированные конфиги сохраняются рядом. Общий лог:
# results/logs/run_followup_<timestamp>.log
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
cd "${PROJECT_ROOT}"
PYTHON="${PYTHON:-python}"
if [[ "${PYTHON}" == */* ]]; then
PYTHON_DIR="$(cd "$(dirname "${PYTHON}")" && pwd)"
PYTHON="${PYTHON_DIR}/$(basename "${PYTHON}")"
fi
if ! "${PYTHON}" -c "import sklearn, pandas, nltk" >/dev/null 2>&1; then
echo "ОШИБКА: в интерпретаторе ${PYTHON} не установлены зависимости."
echo "Выполните: ${PYTHON} -m pip install -r requirements.txt"
exit 1
fi
RESULTS_DIR="${PROJECT_ROOT}/results"
# FOLLOWUP_DIR можно переопределить, чтобы прогнать серию в отдельный каталог
# (например, для дымового теста) и не смешивать с основными результатами.
FOLLOWUP_DIR="${FOLLOWUP_DIR:-${RESULTS_DIR}/followup}"
CONFIG_DIR="${FOLLOWUP_DIR}/configs"
COMPARISON_DIR="${FOLLOWUP_DIR}/comparisons"
LOG_DIR="${RESULTS_DIR}/logs"
MANIFEST="${FOLLOWUP_DIR}/manifest.tsv"
REPEATS_MAIN="${REPEATS_MAIN:-5}"
REPEATS_GRID="${REPEATS_GRID:-3}"
REPEATS_SWEEP="${REPEATS_SWEEP:-3}"
REPEATS_NGRAM="${REPEATS_NGRAM:-3}"
REPEATS_FINAL="${REPEATS_FINAL:-10}"
RUN_GRID="${RUN_GRID:-1}"
RUN_SEEDS="${RUN_SEEDS:-1}"
RUN_FINAL="${RUN_FINAL:-1}"
RUN_CONTROLS="${RUN_CONTROLS:-1}"
SKIP_TESTS="${SKIP_TESTS:-0}"
FORCE="${FORCE:-0}"
SEED="${SEED:-42}"
if [[ "${QUICK:-0}" == "1" ]]; then
REPEATS_MAIN=1
REPEATS_GRID=1
REPEATS_SWEEP=1
REPEATS_NGRAM=1
REPEATS_FINAL=1
RUN_GRID=0
RUN_SEEDS=0
fi
mkdir -p "${FOLLOWUP_DIR}" "${CONFIG_DIR}" "${COMPARISON_DIR}" "${LOG_DIR}"
if [[ ! -f "${MANIFEST}" ]]; then
printf 'label\toriginal_dir\n' > "${MANIFEST}"
fi
RUN_ID="${RUN_ID:-$(date +%Y%m%d_%H%M%S)}"
LOG_FILE="${LOG_DIR}/run_followup_${RUN_ID}.log"
exec > >(tee -a "${LOG_FILE}") 2>&1
step() {
echo
echo "============================================================"
echo "== $*"
echo "============================================================"
}
run() {
echo
echo ">>> $*"
"$@"
}
write_config() {
local name="$1"
cat > "${CONFIG_DIR}/${name}.toml"
}
# Запускает команду, находит созданный ею каталог в results/ и переносит его
# в results/followup/<label>. Так у каждого эксперимента появляется
# стабильное имя, а command.txt фиксирует точную команду.
run_labeled() {
local label="$1"
shift
local before after dir dest
before="$(mktemp)"
after="$(mktemp)"
find "${RESULTS_DIR}" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | sort > "${before}"
run "$@"
find "${RESULTS_DIR}" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | sort > "${after}"
dir="$(comm -13 "${before}" "${after}" | head -n 1)"
rm -f "${before}" "${after}"
if [[ -z "${dir}" ]]; then
echo "ОШИБКА: команда не создала каталог результатов (${label})" >&2
return 1
fi
dest="${FOLLOWUP_DIR}/${label}"
if [[ -e "${dest}" ]]; then
if [[ "${FORCE}" == "1" ]]; then
rm -rf "${dest}"
else
echo "ОШИБКА: ${dest} уже существует. Задайте FORCE=1 или удалите каталог." >&2
return 1
fi
fi
mv "${RESULTS_DIR}/${dir}" "${dest}"
printf '%q ' "$@" > "${dest}/command.txt"
printf '\n' >> "${dest}/command.txt"
printf '%s\t%s\n' "${label}" "${dir}" >> "${MANIFEST}"
echo "[followup] ${label} -> ${dest}"
}
latest_dir() {
local prefix="$1"
ls -1dt "${RESULTS_DIR}/${prefix}"_* 2>/dev/null | head -n 1 || true
}
echo "Начало follow-up серии: $(date)"
echo "Интерпретатор: ${PYTHON}"
echo "Повторы: main=${REPEATS_MAIN}, grid=${REPEATS_GRID}, sweep=${REPEATS_SWEEP}, ngram=${REPEATS_NGRAM}, final=${REPEATS_FINAL}"
echo "Тяжёлые стадии: grid=${RUN_GRID}, seeds=${RUN_SEEDS}, final=${RUN_FINAL}, controls=${RUN_CONTROLS}"
echo "Каталог результатов: ${FOLLOWUP_DIR}"
echo "Лог: ${LOG_FILE}"
# ---------------------------------------------------------------------------
# Конфиги признакового пространства. CLI не умеет менять lsa_components,
# min_df, n-gram ranges и groups, поэтому параметры задаются через TOML.
# ---------------------------------------------------------------------------
step "0. Генерация конфигов"
write_config lsa_content_k300 <<'EOF'
[experiment]
preset = "lsa_content"
model = "logreg"
[features]
lsa_components = 300
EOF
write_config lsa_content_k400 <<'EOF'
[experiment]
preset = "lsa_content"
model = "logreg"
[features]
lsa_components = 400
EOF
write_config lsa_char_k300 <<'EOF'
[experiment]
preset = "lsa_char"
model = "logreg"
[features]
lsa_components = 300
EOF
write_config lsa_pos_k300 <<'EOF'
[experiment]
preset = "lsa_pos"
model = "logreg"
[features]
lsa_components = 300
EOF
write_config stylometry_pos_content300 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
lsa_variants = ["content"]
lsa_components = 300
EOF
write_config stylometry_pos_pos300 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
lsa_variants = ["pos"]
lsa_components = 300
EOF
write_config stylometry_pos_lsa_k300 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
lsa_variants = ["word", "pos"]
lsa_components = 300
EOF
write_config groups_lean1 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
groups = ["length", "lexical", "vocabulary", "characters", "punctuation", "pronouns", "syntax", "readability", "dialogue"]
EOF
write_config groups_lean2 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
groups = ["length", "lexical", "characters", "punctuation", "pronouns", "syntax", "dialogue"]
EOF
write_config groups_lean2_content300 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
groups = ["length", "lexical", "characters", "punctuation", "pronouns", "syntax", "dialogue"]
lsa_variants = ["content"]
lsa_components = 300
EOF
write_config min_df2 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
min_df = 2
EOF
write_config min_df5 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
min_df = 5
EOF
write_config char_max10000 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
char_max_features = 10000
EOF
write_config char_ngram_2_4 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
char_ngram_range = [2, 4]
EOF
write_config char_ngram_3_5 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
char_ngram_range = [3, 5]
EOF
write_config pos_ngram_1_2 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
pos_ngram_range = [1, 2]
EOF
write_config pos_ngram_1_4 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
pos_ngram_range = [1, 4]
EOF
write_config pos_max1000 <<'EOF'
[experiment]
preset = "stylometry_pos_lsa"
model = "logreg"
[features]
pos_max_features = 1000
EOF
echo "Сгенерировано конфигов: $(find "${CONFIG_DIR}" -name '*.toml' | wc -l)"
# ---------------------------------------------------------------------------
# F0. Проверка окружения и воспроизведение опорных результатов.
# ---------------------------------------------------------------------------
step "F0. Проверка окружения и опорные запуски"
run "${PYTHON}" scripts/check_environment.py
if [[ "${SKIP_TESTS}" != "1" ]]; then
run "${PYTHON}" -m pytest -q
fi
run_labeled ref_logreg \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled ref_nested_logreg \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model logreg --folds 5 --nested --seed "${SEED}"
run_labeled ref_no_lsa \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled ref_stylometry \
"${PYTHON}" -m stylometry.cli run --preset stylometry \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
# ---------------------------------------------------------------------------
# F1. Сравнение моделей на одном признаковом пространстве, 5 повторов.
# ---------------------------------------------------------------------------
step "F1. Сравнение моделей (5 повторов + nested)"
run_labeled model_rf_default \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled model_et_default \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model extra_trees --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled model_svc_default \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model linear_svc --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled model_knn_default \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model knn --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}"
run_labeled model_rf_nested \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --nested --seed "${SEED}"
run_labeled model_et_nested \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model extra_trees --folds 5 --nested --seed "${SEED}"
run_labeled model_svc_nested \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model linear_svc --folds 5 --nested --seed "${SEED}"
# ---------------------------------------------------------------------------
# F2. Сетка вокруг лучшей модели (random forest по умолчанию).
# ---------------------------------------------------------------------------
if [[ "${RUN_GRID}" == "1" ]]; then
step "F2. Сетка random forest / extra trees"
for depth in None 30 40; do
for leaf in 1 2 3; do
if [[ "${depth}" == "None" && "${leaf}" == "1" ]]; then
continue # это model_rf_default
fi
if [[ "${depth}" == "None" ]]; then
depth_json="null"
else
depth_json="${depth}"
fi
run_labeled "rf_d${depth}_l${leaf}" \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params "{\"max_depth\": ${depth_json}, \"min_samples_leaf\": ${leaf}}"
done
done
for max_features in 0.05 0.1; do
for n_estimators in 400 800; do
run_labeled "rf_mf${max_features}_n${n_estimators}" \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params "{\"max_features\": ${max_features}, \"n_estimators\": ${n_estimators}, \"max_depth\": null, \"min_samples_leaf\": 1}"
done
done
run_labeled rf_n800_sqrt \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params '{"n_estimators": 800, "max_depth": null, "min_samples_leaf": 1}'
run_labeled rf_cw_balanced \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params '{"class_weight": "balanced"}'
run_labeled rf_cw_none \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params '{"class_weight": null}'
run_labeled et_dNone_l2 \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model extra_trees --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params '{"max_depth": null, "min_samples_leaf": 2}'
run_labeled et_d30_l1 \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model extra_trees --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params '{"max_depth": 30, "min_samples_leaf": 1}'
run_labeled et_d30_l2 \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model extra_trees --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
--model-params '{"max_depth": 30, "min_samples_leaf": 2}'
# Лучшая точка сетки проверяется на 5 повторах. Это разведочный выбор:
# для честной итоговой оценки нужен nested CV (см. final_rf_nested и
# обсуждение в FOLLOWUP_EXPERIMENTS.md).
step "F2.1. Лучшая точка сетки RF на 5 повторах"
BEST_JSON="$("${PYTHON}" - "${FOLLOWUP_DIR}" <<'PY'
import glob
import json
import os
import sys
import pandas as pd
root = sys.argv[1]
candidates = []
for path in glob.glob(os.path.join(root, "rf_*")):
metadata_path = os.path.join(path, "run_metadata.json")
pooled_path = os.path.join(path, "pooled_metrics.csv")
if not (os.path.exists(metadata_path) and os.path.exists(pooled_path)):
continue
with open(metadata_path, encoding="utf-8") as stream:
metadata = json.load(stream)
pooled = pd.read_csv(pooled_path)
params = metadata.get("config", {}).get("model_params") or {}
candidates.append(
(float(pooled["macro_f1"].mean()), os.path.basename(path), json.dumps(params, ensure_ascii=False))
)
if candidates:
candidates.sort(reverse=True)
print(f"{candidates[0][1]}\t{candidates[0][2]}")
PY
)"
if [[ -n "${BEST_JSON}" ]]; then
BEST_LABEL="${BEST_JSON%%$'\t'*}"
BEST_PARAMS="${BEST_JSON#*$'\t'}"
echo "[followup] лучшая точка сетки: ${BEST_LABEL}, параметры: ${BEST_PARAMS}"
run_labeled rf_grid_best_5 \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
--model-params "${BEST_PARAMS}"
else
echo "[followup] не удалось выбрать лучшую точку сетки RF"
fi
else
echo
echo "F2 пропущена (RUN_GRID=${RUN_GRID})."
fi
# ---------------------------------------------------------------------------
# F3. LSA: тема против стиля. Перебор k и подтверждение на 5 повторах.
# ---------------------------------------------------------------------------
step "F3. LSA: варианты, число компонент и объединение с признаками"
run_labeled lsa_sweep \
"${PYTHON}" -m stylometry.cli lsa-sweep \
--variants content,char,word,pos,fw --components 100,200,300,400 \
--model logreg --folds 5 --repeats "${REPEATS_SWEEP}" --seed "${SEED}"
run_labeled lsa_content_k300 \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_content_k300.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled lsa_content_k400 \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_content_k400.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled lsa_char_k300 \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_char_k300.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled lsa_pos_k300 \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_pos_k300.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled stylometry_pos_content300 \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/stylometry_pos_content300.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled stylometry_pos_pos300 \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/stylometry_pos_pos300.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled stylometry_pos_lsa_k300 \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/stylometry_pos_lsa_k300.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
# ---------------------------------------------------------------------------
# F4. Абляция групп признаков на 5 повторах и «худые» наборы.
# ---------------------------------------------------------------------------
step "F4. Абляция групп признаков и lean-наборы"
run_labeled ablate_full \
"${PYTHON}" -m stylometry.cli ablate --preset stylometry_pos_lsa \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
for name in groups_lean1 groups_lean2 groups_lean2_content300; do
run_labeled "${name}" \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/${name}.toml" \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
done
run_labeled groups_lean1_rf \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/groups_lean1.toml" \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled groups_lean2_rf \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/groups_lean2.toml" \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
# ---------------------------------------------------------------------------
# F5. Настройка символьных и POS n-грамм.
# ---------------------------------------------------------------------------
step "F5. Символьные и POS n-граммы"
for name in min_df2 min_df5 char_max10000 char_ngram_2_4 char_ngram_3_5; do
run_labeled "${name}" \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/${name}.toml" \
--model logreg --folds 5 --repeats "${REPEATS_NGRAM}" --seed "${SEED}"
done
for name in pos_ngram_1_2 pos_ngram_1_4 pos_max1000; do
run_labeled "${name}" \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/${name}.toml" \
--model logreg --folds 5 --repeats "${REPEATS_NGRAM}" --seed "${SEED}"
done
run_labeled min_df2_rf \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/min_df2.toml" \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled char_max10000_rf \
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/char_max10000.toml" \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
# ---------------------------------------------------------------------------
# F6. Балансировка классов.
# ---------------------------------------------------------------------------
step "F6. class_weight: balanced против none"
run_labeled classweight_logreg_none \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
--model-params '{"class_weight": null}'
run_labeled classweight_svc_none \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model linear_svc --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
--model-params '{"class_weight": null}'
run_labeled classweight_rf_none \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
--model-params '{"class_weight": null}'
run_labeled classweight_rf_balanced \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
--model-params '{"class_weight": "balanced"}'
# ---------------------------------------------------------------------------
# F7. Устойчивость к seed.
# ---------------------------------------------------------------------------
if [[ "${RUN_SEEDS}" == "1" ]]; then
step "F7. Устойчивость к seed (logreg и random forest)"
for seed in 7 2024 1337; do
run_labeled "seed_logreg_${seed}" \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${seed}"
run_labeled "seed_rf_${seed}" \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${seed}" \
--model-params "{\"random_state\": ${seed}}"
done
else
echo
echo "F7 пропущена (RUN_SEEDS=${RUN_SEEDS})."
fi
# ---------------------------------------------------------------------------
# F8. Финальная модель, отчёты и независимые проверки.
# ---------------------------------------------------------------------------
if [[ "${RUN_FINAL}" == "1" ]]; then
step "F8. Финальная модель и отчёты"
run_labeled final_rf \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --repeats "${REPEATS_FINAL}" --seed "${SEED}"
run_labeled final_rf_nested \
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
--model random_forest --folds 5 --nested --seed "${SEED}"
FINAL_DIR="${FOLLOWUP_DIR}/final_rf"
if [[ -d "${FINAL_DIR}" ]]; then
run "${PYTHON}" -m stylometry.cli length-report \
--run-dir "${FINAL_DIR}" --seed "${SEED}"
for group in gender age_group species; do
run "${PYTHON}" -m stylometry.cli subgroup-report \
--run-dir "${FINAL_DIR}" --group "${group}"
done
run "${PYTHON}" -m stylometry.cli compare \
--run-a "${FINAL_DIR}" --run-b "${FOLLOWUP_DIR}/ref_logreg" \
--out "${COMPARISON_DIR}/final_rf_vs_ref_logreg.json" --seed "${SEED}"
run "${PYTHON}" -m stylometry.cli compare \
--run-a "${FINAL_DIR}" --run-b "${FOLLOWUP_DIR}/ref_no_lsa" \
--out "${COMPARISON_DIR}/final_rf_vs_ref_no_lsa.json" --seed "${SEED}"
fi
run_labeled verify_full_42 \
"${PYTHON}" -m stylometry.cli verify --preset stylometry_pos_lsa \
--folds 5 --seed 42
run_labeled verify_full_7 \
"${PYTHON}" -m stylometry.cli verify --preset stylometry_pos_lsa \
--folds 5 --seed 7
run_labeled verify_full_2024 \
"${PYTHON}" -m stylometry.cli verify --preset stylometry_pos_lsa \
--folds 5 --seed 2024
run_labeled delta_cosine \
"${PYTHON}" -m stylometry.cli delta --features 500,1000,2000 \
--metrics cosine --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}"
for clusters in 3 5 8; do
run_labeled "cluster_k${clusters}" \
"${PYTHON}" -m stylometry.cli cluster --preset stylometry_pos_lsa \
--clusters "${clusters}" --seed "${SEED}"
done
else
echo
echo "F8 пропущена (RUN_FINAL=${RUN_FINAL})."
fi
# ---------------------------------------------------------------------------
# F9. Контроли: нижние границы, near-duplicates и перестановка меток.
# ---------------------------------------------------------------------------
if [[ "${RUN_CONTROLS}" == "1" ]]; then
step "F9. Контроли"
run_labeled control_length_logreg \
"${PYTHON}" -m stylometry.cli run --preset length \
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
run_labeled control_length_dummy \
"${PYTHON}" -m stylometry.cli run --preset length \
--model dummy --folds 5 --repeats 1 --seed "${SEED}"
step "F9.1. Поиск near-duplicates"
"${PYTHON}" - "${FOLLOWUP_DIR}" <<'PY'
from pathlib import Path
import sys
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from stylometry.datasets import load_feature_frame
frame = load_feature_frame(verbose=False)
matrix = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), min_df=1).fit_transform(
frame["text"]
)
similarity = cosine_similarity(matrix)
rows = np.triu_indices_from(similarity, k=1)
values = similarity[rows]
threshold = 0.8
mask = values >= threshold
pairs = pd.DataFrame(
{
"line_id_a": frame.iloc[rows[0][mask]]["line_id"].to_numpy(),
"character_a": frame.iloc[rows[0][mask]]["character"].to_numpy(),
"text_a": frame.iloc[rows[0][mask]]["text"].to_numpy(),
"line_id_b": frame.iloc[rows[1][mask]]["line_id"].to_numpy(),
"character_b": frame.iloc[rows[1][mask]]["character"].to_numpy(),
"text_b": frame.iloc[rows[1][mask]]["text"].to_numpy(),
"cosine": values[mask],
"same_character": frame.iloc[rows[0][mask]]["character"].to_numpy()
== frame.iloc[rows[1][mask]]["character"].to_numpy(),
}
).sort_values("cosine", ascending=False)
out_dir = Path(sys.argv[1]) / "negative_control"
out_dir.mkdir(parents=True, exist_ok=True)
pairs.to_csv(out_dir / "near_duplicates.csv", index=False, encoding="utf-8-sig")
print(f"[near-duplicates] пар с cosine >= {threshold}: {len(pairs)}")
if not pairs.empty:
print(pairs.head(10).to_string(index=False))
PY
step "F9.2. Контроль перестановкой меток"
"${PYTHON}" - "${FOLLOWUP_DIR}" <<'PY'
from pathlib import Path
import sys
import numpy as np
import pandas as pd
from stylometry.cv import run_repeated_cv
from stylometry.datasets import load_feature_frame
from stylometry.pipelines import build_pipeline, preset_config
frame = load_feature_frame(verbose=False)
y = frame["character"].to_numpy()
config = preset_config("stylometry_pos_lsa")
rows = []
for seed in (42, 7, 2024):
rng = np.random.default_rng(seed)
shuffled = rng.permutation(y)
pipeline = build_pipeline(config, "logreg")
result = run_repeated_cv(
pipeline,
frame,
shuffled,
n_splits=5,
n_repeats=1,
seed=seed,
model_name="permuted_logreg",
verbose=False,
)
pooled, _ = result.pooled_summary()
row = pooled.iloc[0].to_dict()
row["permutation_seed"] = seed
rows.append(row)
out_dir = Path(sys.argv[1]) / "negative_control"
out_dir.mkdir(parents=True, exist_ok=True)
table = pd.DataFrame(rows)
table.to_csv(out_dir / "permutation_logreg.csv", index=False, encoding="utf-8-sig")
print(table[["permutation_seed", "macro_f1", "accuracy", "top3_accuracy"]].to_string(index=False))
print(f"[permutation] средний macro-F1: {table['macro_f1'].mean():.4f}")
PY
else
echo
echo "F9 пропущена (RUN_CONTROLS=${RUN_CONTROLS})."
fi
# ---------------------------------------------------------------------------
# Сводка по всем запускам и парные сравнения по повторам.
# ---------------------------------------------------------------------------
step "Сводка follow-up"
PAIRS=(
"model_rf_default:ref_logreg"
"model_rf_default:ref_no_lsa"
"model_rf_default:model_svc_default"
"rf_grid_best_5:model_rf_default"
"model_svc_default:ref_logreg"
"ref_no_lsa:ref_logreg"
"stylometry_pos_content300:ref_no_lsa"
"stylometry_pos_pos300:ref_no_lsa"
"stylometry_pos_lsa_k300:ref_logreg"
"groups_lean1:ref_logreg"
"groups_lean2:ref_logreg"
"groups_lean2_content300:ref_logreg"
"groups_lean1_rf:model_rf_default"
"groups_lean2_rf:model_rf_default"
"min_df2:ref_logreg"
"char_max10000:ref_logreg"
"char_ngram_2_4:ref_logreg"
"char_ngram_3_5:ref_logreg"
"pos_ngram_1_2:ref_logreg"
"pos_ngram_1_4:ref_logreg"
"pos_max1000:ref_logreg"
"min_df2_rf:model_rf_default"
"char_max10000_rf:model_rf_default"
"classweight_logreg_none:ref_logreg"
"classweight_svc_none:model_svc_default"
"classweight_rf_none:model_rf_default"
"classweight_rf_balanced:model_rf_default"
"final_rf:ref_logreg"
"final_rf:ref_no_lsa"
"final_rf:model_rf_default"
"final_rf:final_rf_nested"
"control_length_logreg:ref_logreg"
)
run "${PYTHON}" scripts/analyze_followup.py \
--followup-dir "${FOLLOWUP_DIR}" \
--pairs "${PAIRS[@]}" \
--n-boot 1000 --seed "${SEED}"
step "Готово"
echo "Конец follow-up серии: $(date)"
echo
echo "Результаты: ${FOLLOWUP_DIR}"
echo "Сводка: ${FOLLOWUP_DIR}/summary.csv"
echo "Сравнения: ${FOLLOWUP_DIR}/comparisons.csv"
echo "Лог: ${LOG_FILE}"