825 lines
30 KiB
Bash
Executable File
825 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Follow-up серия экспериментов по итогам предварительных прогонов.
|
||
#
|
||
# Запуск из каталога lab1:
|
||
# bash scripts/run_followup_experiments.sh
|
||
#
|
||
# С другим интерпретатором:
|
||
# PYTHON=../.venv/bin/python bash scripts/run_followup_experiments.sh
|
||
#
|
||
# Быстрая проверка (1 повтор, без сетки и seed-робастности):
|
||
# QUICK=1 bash scripts/run_followup_experiments.sh
|
||
#
|
||
# Только воспроизведение и финальная модель, без тяжёлых стадий:
|
||
# RUN_GRID=0 RUN_SEEDS=0 bash scripts/run_followup_experiments.sh
|
||
#
|
||
# Все результаты складываются в results/followup/<label>/, команды и
|
||
# сгенерированные конфиги сохраняются рядом. Общий лог:
|
||
# results/logs/run_followup_<timestamp>.log
|
||
|
||
set -euo pipefail
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||
cd "${PROJECT_ROOT}"
|
||
|
||
PYTHON="${PYTHON:-python}"
|
||
if [[ "${PYTHON}" == */* ]]; then
|
||
PYTHON_DIR="$(cd "$(dirname "${PYTHON}")" && pwd)"
|
||
PYTHON="${PYTHON_DIR}/$(basename "${PYTHON}")"
|
||
fi
|
||
|
||
if ! "${PYTHON}" -c "import sklearn, pandas, nltk" >/dev/null 2>&1; then
|
||
echo "ОШИБКА: в интерпретаторе ${PYTHON} не установлены зависимости."
|
||
echo "Выполните: ${PYTHON} -m pip install -r requirements.txt"
|
||
exit 1
|
||
fi
|
||
|
||
RESULTS_DIR="${PROJECT_ROOT}/results"
|
||
# FOLLOWUP_DIR можно переопределить, чтобы прогнать серию в отдельный каталог
|
||
# (например, для дымового теста) и не смешивать с основными результатами.
|
||
FOLLOWUP_DIR="${FOLLOWUP_DIR:-${RESULTS_DIR}/followup}"
|
||
CONFIG_DIR="${FOLLOWUP_DIR}/configs"
|
||
COMPARISON_DIR="${FOLLOWUP_DIR}/comparisons"
|
||
LOG_DIR="${RESULTS_DIR}/logs"
|
||
MANIFEST="${FOLLOWUP_DIR}/manifest.tsv"
|
||
|
||
REPEATS_MAIN="${REPEATS_MAIN:-5}"
|
||
REPEATS_GRID="${REPEATS_GRID:-3}"
|
||
REPEATS_SWEEP="${REPEATS_SWEEP:-3}"
|
||
REPEATS_NGRAM="${REPEATS_NGRAM:-3}"
|
||
REPEATS_FINAL="${REPEATS_FINAL:-10}"
|
||
RUN_GRID="${RUN_GRID:-1}"
|
||
RUN_SEEDS="${RUN_SEEDS:-1}"
|
||
RUN_FINAL="${RUN_FINAL:-1}"
|
||
RUN_CONTROLS="${RUN_CONTROLS:-1}"
|
||
SKIP_TESTS="${SKIP_TESTS:-0}"
|
||
FORCE="${FORCE:-0}"
|
||
SEED="${SEED:-42}"
|
||
|
||
if [[ "${QUICK:-0}" == "1" ]]; then
|
||
REPEATS_MAIN=1
|
||
REPEATS_GRID=1
|
||
REPEATS_SWEEP=1
|
||
REPEATS_NGRAM=1
|
||
REPEATS_FINAL=1
|
||
RUN_GRID=0
|
||
RUN_SEEDS=0
|
||
fi
|
||
|
||
mkdir -p "${FOLLOWUP_DIR}" "${CONFIG_DIR}" "${COMPARISON_DIR}" "${LOG_DIR}"
|
||
if [[ ! -f "${MANIFEST}" ]]; then
|
||
printf 'label\toriginal_dir\n' > "${MANIFEST}"
|
||
fi
|
||
|
||
RUN_ID="${RUN_ID:-$(date +%Y%m%d_%H%M%S)}"
|
||
LOG_FILE="${LOG_DIR}/run_followup_${RUN_ID}.log"
|
||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||
|
||
step() {
|
||
echo
|
||
echo "============================================================"
|
||
echo "== $*"
|
||
echo "============================================================"
|
||
}
|
||
|
||
run() {
|
||
echo
|
||
echo ">>> $*"
|
||
"$@"
|
||
}
|
||
|
||
write_config() {
|
||
local name="$1"
|
||
cat > "${CONFIG_DIR}/${name}.toml"
|
||
}
|
||
|
||
# Запускает команду, находит созданный ею каталог в results/ и переносит его
|
||
# в results/followup/<label>. Так у каждого эксперимента появляется
|
||
# стабильное имя, а command.txt фиксирует точную команду.
|
||
run_labeled() {
|
||
local label="$1"
|
||
shift
|
||
local before after dir dest
|
||
before="$(mktemp)"
|
||
after="$(mktemp)"
|
||
find "${RESULTS_DIR}" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | sort > "${before}"
|
||
run "$@"
|
||
find "${RESULTS_DIR}" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | sort > "${after}"
|
||
dir="$(comm -13 "${before}" "${after}" | head -n 1)"
|
||
rm -f "${before}" "${after}"
|
||
if [[ -z "${dir}" ]]; then
|
||
echo "ОШИБКА: команда не создала каталог результатов (${label})" >&2
|
||
return 1
|
||
fi
|
||
dest="${FOLLOWUP_DIR}/${label}"
|
||
if [[ -e "${dest}" ]]; then
|
||
if [[ "${FORCE}" == "1" ]]; then
|
||
rm -rf "${dest}"
|
||
else
|
||
echo "ОШИБКА: ${dest} уже существует. Задайте FORCE=1 или удалите каталог." >&2
|
||
return 1
|
||
fi
|
||
fi
|
||
mv "${RESULTS_DIR}/${dir}" "${dest}"
|
||
printf '%q ' "$@" > "${dest}/command.txt"
|
||
printf '\n' >> "${dest}/command.txt"
|
||
printf '%s\t%s\n' "${label}" "${dir}" >> "${MANIFEST}"
|
||
echo "[followup] ${label} -> ${dest}"
|
||
}
|
||
|
||
latest_dir() {
|
||
local prefix="$1"
|
||
ls -1dt "${RESULTS_DIR}/${prefix}"_* 2>/dev/null | head -n 1 || true
|
||
}
|
||
|
||
echo "Начало follow-up серии: $(date)"
|
||
echo "Интерпретатор: ${PYTHON}"
|
||
echo "Повторы: main=${REPEATS_MAIN}, grid=${REPEATS_GRID}, sweep=${REPEATS_SWEEP}, ngram=${REPEATS_NGRAM}, final=${REPEATS_FINAL}"
|
||
echo "Тяжёлые стадии: grid=${RUN_GRID}, seeds=${RUN_SEEDS}, final=${RUN_FINAL}, controls=${RUN_CONTROLS}"
|
||
echo "Каталог результатов: ${FOLLOWUP_DIR}"
|
||
echo "Лог: ${LOG_FILE}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Конфиги признакового пространства. CLI не умеет менять lsa_components,
|
||
# min_df, n-gram ranges и groups, поэтому параметры задаются через TOML.
|
||
# ---------------------------------------------------------------------------
|
||
step "0. Генерация конфигов"
|
||
|
||
write_config lsa_content_k300 <<'EOF'
|
||
[experiment]
|
||
preset = "lsa_content"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
lsa_components = 300
|
||
EOF
|
||
|
||
write_config lsa_content_k400 <<'EOF'
|
||
[experiment]
|
||
preset = "lsa_content"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
lsa_components = 400
|
||
EOF
|
||
|
||
write_config lsa_char_k300 <<'EOF'
|
||
[experiment]
|
||
preset = "lsa_char"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
lsa_components = 300
|
||
EOF
|
||
|
||
write_config lsa_pos_k300 <<'EOF'
|
||
[experiment]
|
||
preset = "lsa_pos"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
lsa_components = 300
|
||
EOF
|
||
|
||
write_config stylometry_pos_content300 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
lsa_variants = ["content"]
|
||
lsa_components = 300
|
||
EOF
|
||
|
||
write_config stylometry_pos_pos300 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
lsa_variants = ["pos"]
|
||
lsa_components = 300
|
||
EOF
|
||
|
||
write_config stylometry_pos_lsa_k300 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
lsa_variants = ["word", "pos"]
|
||
lsa_components = 300
|
||
EOF
|
||
|
||
write_config groups_lean1 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
groups = ["length", "lexical", "vocabulary", "characters", "punctuation", "pronouns", "syntax", "readability", "dialogue"]
|
||
EOF
|
||
|
||
write_config groups_lean2 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
groups = ["length", "lexical", "characters", "punctuation", "pronouns", "syntax", "dialogue"]
|
||
EOF
|
||
|
||
write_config groups_lean2_content300 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
groups = ["length", "lexical", "characters", "punctuation", "pronouns", "syntax", "dialogue"]
|
||
lsa_variants = ["content"]
|
||
lsa_components = 300
|
||
EOF
|
||
|
||
write_config min_df2 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
min_df = 2
|
||
EOF
|
||
|
||
write_config min_df5 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
min_df = 5
|
||
EOF
|
||
|
||
write_config char_max10000 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
char_max_features = 10000
|
||
EOF
|
||
|
||
write_config char_ngram_2_4 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
char_ngram_range = [2, 4]
|
||
EOF
|
||
|
||
write_config char_ngram_3_5 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
char_ngram_range = [3, 5]
|
||
EOF
|
||
|
||
write_config pos_ngram_1_2 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
pos_ngram_range = [1, 2]
|
||
EOF
|
||
|
||
write_config pos_ngram_1_4 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
pos_ngram_range = [1, 4]
|
||
EOF
|
||
|
||
write_config pos_max1000 <<'EOF'
|
||
[experiment]
|
||
preset = "stylometry_pos_lsa"
|
||
model = "logreg"
|
||
|
||
[features]
|
||
pos_max_features = 1000
|
||
EOF
|
||
|
||
echo "Сгенерировано конфигов: $(find "${CONFIG_DIR}" -name '*.toml' | wc -l)"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F0. Проверка окружения и воспроизведение опорных результатов.
|
||
# ---------------------------------------------------------------------------
|
||
step "F0. Проверка окружения и опорные запуски"
|
||
|
||
run "${PYTHON}" scripts/check_environment.py
|
||
if [[ "${SKIP_TESTS}" != "1" ]]; then
|
||
run "${PYTHON}" -m pytest -q
|
||
fi
|
||
|
||
run_labeled ref_logreg \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled ref_nested_logreg \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model logreg --folds 5 --nested --seed "${SEED}"
|
||
|
||
run_labeled ref_no_lsa \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled ref_stylometry \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F1. Сравнение моделей на одном признаковом пространстве, 5 повторов.
|
||
# ---------------------------------------------------------------------------
|
||
step "F1. Сравнение моделей (5 повторов + nested)"
|
||
|
||
run_labeled model_rf_default \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled model_et_default \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model extra_trees --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled model_svc_default \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model linear_svc --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled model_knn_default \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model knn --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}"
|
||
|
||
run_labeled model_rf_nested \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --nested --seed "${SEED}"
|
||
|
||
run_labeled model_et_nested \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model extra_trees --folds 5 --nested --seed "${SEED}"
|
||
|
||
run_labeled model_svc_nested \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model linear_svc --folds 5 --nested --seed "${SEED}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F2. Сетка вокруг лучшей модели (random forest по умолчанию).
|
||
# ---------------------------------------------------------------------------
|
||
if [[ "${RUN_GRID}" == "1" ]]; then
|
||
step "F2. Сетка random forest / extra trees"
|
||
|
||
for depth in None 30 40; do
|
||
for leaf in 1 2 3; do
|
||
if [[ "${depth}" == "None" && "${leaf}" == "1" ]]; then
|
||
continue # это model_rf_default
|
||
fi
|
||
if [[ "${depth}" == "None" ]]; then
|
||
depth_json="null"
|
||
else
|
||
depth_json="${depth}"
|
||
fi
|
||
run_labeled "rf_d${depth}_l${leaf}" \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params "{\"max_depth\": ${depth_json}, \"min_samples_leaf\": ${leaf}}"
|
||
done
|
||
done
|
||
|
||
for max_features in 0.05 0.1; do
|
||
for n_estimators in 400 800; do
|
||
run_labeled "rf_mf${max_features}_n${n_estimators}" \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params "{\"max_features\": ${max_features}, \"n_estimators\": ${n_estimators}, \"max_depth\": null, \"min_samples_leaf\": 1}"
|
||
done
|
||
done
|
||
|
||
run_labeled rf_n800_sqrt \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params '{"n_estimators": 800, "max_depth": null, "min_samples_leaf": 1}'
|
||
|
||
run_labeled rf_cw_balanced \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params '{"class_weight": "balanced"}'
|
||
|
||
run_labeled rf_cw_none \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params '{"class_weight": null}'
|
||
|
||
run_labeled et_dNone_l2 \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model extra_trees --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params '{"max_depth": null, "min_samples_leaf": 2}'
|
||
|
||
run_labeled et_d30_l1 \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model extra_trees --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params '{"max_depth": 30, "min_samples_leaf": 1}'
|
||
|
||
run_labeled et_d30_l2 \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model extra_trees --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}" \
|
||
--model-params '{"max_depth": 30, "min_samples_leaf": 2}'
|
||
|
||
# Лучшая точка сетки проверяется на 5 повторах. Это разведочный выбор:
|
||
# для честной итоговой оценки нужен nested CV (см. final_rf_nested и
|
||
# обсуждение в FOLLOWUP_EXPERIMENTS.md).
|
||
step "F2.1. Лучшая точка сетки RF на 5 повторах"
|
||
BEST_JSON="$("${PYTHON}" - "${FOLLOWUP_DIR}" <<'PY'
|
||
import glob
|
||
import json
|
||
import os
|
||
import sys
|
||
|
||
import pandas as pd
|
||
|
||
root = sys.argv[1]
|
||
candidates = []
|
||
for path in glob.glob(os.path.join(root, "rf_*")):
|
||
metadata_path = os.path.join(path, "run_metadata.json")
|
||
pooled_path = os.path.join(path, "pooled_metrics.csv")
|
||
if not (os.path.exists(metadata_path) and os.path.exists(pooled_path)):
|
||
continue
|
||
with open(metadata_path, encoding="utf-8") as stream:
|
||
metadata = json.load(stream)
|
||
pooled = pd.read_csv(pooled_path)
|
||
params = metadata.get("config", {}).get("model_params") or {}
|
||
candidates.append(
|
||
(float(pooled["macro_f1"].mean()), os.path.basename(path), json.dumps(params, ensure_ascii=False))
|
||
)
|
||
if candidates:
|
||
candidates.sort(reverse=True)
|
||
print(f"{candidates[0][1]}\t{candidates[0][2]}")
|
||
PY
|
||
)"
|
||
if [[ -n "${BEST_JSON}" ]]; then
|
||
BEST_LABEL="${BEST_JSON%%$'\t'*}"
|
||
BEST_PARAMS="${BEST_JSON#*$'\t'}"
|
||
echo "[followup] лучшая точка сетки: ${BEST_LABEL}, параметры: ${BEST_PARAMS}"
|
||
run_labeled rf_grid_best_5 \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
|
||
--model-params "${BEST_PARAMS}"
|
||
else
|
||
echo "[followup] не удалось выбрать лучшую точку сетки RF"
|
||
fi
|
||
else
|
||
echo
|
||
echo "F2 пропущена (RUN_GRID=${RUN_GRID})."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F3. LSA: тема против стиля. Перебор k и подтверждение на 5 повторах.
|
||
# ---------------------------------------------------------------------------
|
||
step "F3. LSA: варианты, число компонент и объединение с признаками"
|
||
|
||
run_labeled lsa_sweep \
|
||
"${PYTHON}" -m stylometry.cli lsa-sweep \
|
||
--variants content,char,word,pos,fw --components 100,200,300,400 \
|
||
--model logreg --folds 5 --repeats "${REPEATS_SWEEP}" --seed "${SEED}"
|
||
|
||
run_labeled lsa_content_k300 \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_content_k300.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled lsa_content_k400 \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_content_k400.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled lsa_char_k300 \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_char_k300.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled lsa_pos_k300 \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/lsa_pos_k300.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled stylometry_pos_content300 \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/stylometry_pos_content300.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled stylometry_pos_pos300 \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/stylometry_pos_pos300.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled stylometry_pos_lsa_k300 \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/stylometry_pos_lsa_k300.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F4. Абляция групп признаков на 5 повторах и «худые» наборы.
|
||
# ---------------------------------------------------------------------------
|
||
step "F4. Абляция групп признаков и lean-наборы"
|
||
|
||
run_labeled ablate_full \
|
||
"${PYTHON}" -m stylometry.cli ablate --preset stylometry_pos_lsa \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
for name in groups_lean1 groups_lean2 groups_lean2_content300; do
|
||
run_labeled "${name}" \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/${name}.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
done
|
||
|
||
run_labeled groups_lean1_rf \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/groups_lean1.toml" \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled groups_lean2_rf \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/groups_lean2.toml" \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F5. Настройка символьных и POS n-грамм.
|
||
# ---------------------------------------------------------------------------
|
||
step "F5. Символьные и POS n-граммы"
|
||
|
||
for name in min_df2 min_df5 char_max10000 char_ngram_2_4 char_ngram_3_5; do
|
||
run_labeled "${name}" \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/${name}.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_NGRAM}" --seed "${SEED}"
|
||
done
|
||
|
||
for name in pos_ngram_1_2 pos_ngram_1_4 pos_max1000; do
|
||
run_labeled "${name}" \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/${name}.toml" \
|
||
--model logreg --folds 5 --repeats "${REPEATS_NGRAM}" --seed "${SEED}"
|
||
done
|
||
|
||
run_labeled min_df2_rf \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/min_df2.toml" \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled char_max10000_rf \
|
||
"${PYTHON}" -m stylometry.cli run --config "${CONFIG_DIR}/char_max10000.toml" \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F6. Балансировка классов.
|
||
# ---------------------------------------------------------------------------
|
||
step "F6. class_weight: balanced против none"
|
||
|
||
run_labeled classweight_logreg_none \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
|
||
--model-params '{"class_weight": null}'
|
||
|
||
run_labeled classweight_svc_none \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model linear_svc --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
|
||
--model-params '{"class_weight": null}'
|
||
|
||
run_labeled classweight_rf_none \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
|
||
--model-params '{"class_weight": null}'
|
||
|
||
run_labeled classweight_rf_balanced \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}" \
|
||
--model-params '{"class_weight": "balanced"}'
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F7. Устойчивость к seed.
|
||
# ---------------------------------------------------------------------------
|
||
if [[ "${RUN_SEEDS}" == "1" ]]; then
|
||
step "F7. Устойчивость к seed (logreg и random forest)"
|
||
for seed in 7 2024 1337; do
|
||
run_labeled "seed_logreg_${seed}" \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${seed}"
|
||
|
||
run_labeled "seed_rf_${seed}" \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_MAIN}" --seed "${seed}" \
|
||
--model-params "{\"random_state\": ${seed}}"
|
||
done
|
||
else
|
||
echo
|
||
echo "F7 пропущена (RUN_SEEDS=${RUN_SEEDS})."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F8. Финальная модель, отчёты и независимые проверки.
|
||
# ---------------------------------------------------------------------------
|
||
if [[ "${RUN_FINAL}" == "1" ]]; then
|
||
step "F8. Финальная модель и отчёты"
|
||
|
||
run_labeled final_rf \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --repeats "${REPEATS_FINAL}" --seed "${SEED}"
|
||
|
||
run_labeled final_rf_nested \
|
||
"${PYTHON}" -m stylometry.cli run --preset stylometry_pos_lsa \
|
||
--model random_forest --folds 5 --nested --seed "${SEED}"
|
||
|
||
FINAL_DIR="${FOLLOWUP_DIR}/final_rf"
|
||
if [[ -d "${FINAL_DIR}" ]]; then
|
||
run "${PYTHON}" -m stylometry.cli length-report \
|
||
--run-dir "${FINAL_DIR}" --seed "${SEED}"
|
||
for group in gender age_group species; do
|
||
run "${PYTHON}" -m stylometry.cli subgroup-report \
|
||
--run-dir "${FINAL_DIR}" --group "${group}"
|
||
done
|
||
run "${PYTHON}" -m stylometry.cli compare \
|
||
--run-a "${FINAL_DIR}" --run-b "${FOLLOWUP_DIR}/ref_logreg" \
|
||
--out "${COMPARISON_DIR}/final_rf_vs_ref_logreg.json" --seed "${SEED}"
|
||
run "${PYTHON}" -m stylometry.cli compare \
|
||
--run-a "${FINAL_DIR}" --run-b "${FOLLOWUP_DIR}/ref_no_lsa" \
|
||
--out "${COMPARISON_DIR}/final_rf_vs_ref_no_lsa.json" --seed "${SEED}"
|
||
fi
|
||
|
||
run_labeled verify_full_42 \
|
||
"${PYTHON}" -m stylometry.cli verify --preset stylometry_pos_lsa \
|
||
--folds 5 --seed 42
|
||
run_labeled verify_full_7 \
|
||
"${PYTHON}" -m stylometry.cli verify --preset stylometry_pos_lsa \
|
||
--folds 5 --seed 7
|
||
run_labeled verify_full_2024 \
|
||
"${PYTHON}" -m stylometry.cli verify --preset stylometry_pos_lsa \
|
||
--folds 5 --seed 2024
|
||
|
||
run_labeled delta_cosine \
|
||
"${PYTHON}" -m stylometry.cli delta --features 500,1000,2000 \
|
||
--metrics cosine --folds 5 --repeats "${REPEATS_GRID}" --seed "${SEED}"
|
||
|
||
for clusters in 3 5 8; do
|
||
run_labeled "cluster_k${clusters}" \
|
||
"${PYTHON}" -m stylometry.cli cluster --preset stylometry_pos_lsa \
|
||
--clusters "${clusters}" --seed "${SEED}"
|
||
done
|
||
else
|
||
echo
|
||
echo "F8 пропущена (RUN_FINAL=${RUN_FINAL})."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F9. Контроли: нижние границы, near-duplicates и перестановка меток.
|
||
# ---------------------------------------------------------------------------
|
||
if [[ "${RUN_CONTROLS}" == "1" ]]; then
|
||
step "F9. Контроли"
|
||
|
||
run_labeled control_length_logreg \
|
||
"${PYTHON}" -m stylometry.cli run --preset length \
|
||
--model logreg --folds 5 --repeats "${REPEATS_MAIN}" --seed "${SEED}"
|
||
|
||
run_labeled control_length_dummy \
|
||
"${PYTHON}" -m stylometry.cli run --preset length \
|
||
--model dummy --folds 5 --repeats 1 --seed "${SEED}"
|
||
|
||
step "F9.1. Поиск near-duplicates"
|
||
"${PYTHON}" - "${FOLLOWUP_DIR}" <<'PY'
|
||
from pathlib import Path
|
||
import sys
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||
from sklearn.metrics.pairwise import cosine_similarity
|
||
|
||
from stylometry.datasets import load_feature_frame
|
||
|
||
frame = load_feature_frame(verbose=False)
|
||
matrix = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), min_df=1).fit_transform(
|
||
frame["text"]
|
||
)
|
||
similarity = cosine_similarity(matrix)
|
||
rows = np.triu_indices_from(similarity, k=1)
|
||
values = similarity[rows]
|
||
threshold = 0.8
|
||
mask = values >= threshold
|
||
pairs = pd.DataFrame(
|
||
{
|
||
"line_id_a": frame.iloc[rows[0][mask]]["line_id"].to_numpy(),
|
||
"character_a": frame.iloc[rows[0][mask]]["character"].to_numpy(),
|
||
"text_a": frame.iloc[rows[0][mask]]["text"].to_numpy(),
|
||
"line_id_b": frame.iloc[rows[1][mask]]["line_id"].to_numpy(),
|
||
"character_b": frame.iloc[rows[1][mask]]["character"].to_numpy(),
|
||
"text_b": frame.iloc[rows[1][mask]]["text"].to_numpy(),
|
||
"cosine": values[mask],
|
||
"same_character": frame.iloc[rows[0][mask]]["character"].to_numpy()
|
||
== frame.iloc[rows[1][mask]]["character"].to_numpy(),
|
||
}
|
||
).sort_values("cosine", ascending=False)
|
||
out_dir = Path(sys.argv[1]) / "negative_control"
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
pairs.to_csv(out_dir / "near_duplicates.csv", index=False, encoding="utf-8-sig")
|
||
print(f"[near-duplicates] пар с cosine >= {threshold}: {len(pairs)}")
|
||
if not pairs.empty:
|
||
print(pairs.head(10).to_string(index=False))
|
||
PY
|
||
|
||
step "F9.2. Контроль перестановкой меток"
|
||
"${PYTHON}" - "${FOLLOWUP_DIR}" <<'PY'
|
||
from pathlib import Path
|
||
import sys
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
from stylometry.cv import run_repeated_cv
|
||
from stylometry.datasets import load_feature_frame
|
||
from stylometry.pipelines import build_pipeline, preset_config
|
||
|
||
frame = load_feature_frame(verbose=False)
|
||
y = frame["character"].to_numpy()
|
||
config = preset_config("stylometry_pos_lsa")
|
||
rows = []
|
||
for seed in (42, 7, 2024):
|
||
rng = np.random.default_rng(seed)
|
||
shuffled = rng.permutation(y)
|
||
pipeline = build_pipeline(config, "logreg")
|
||
result = run_repeated_cv(
|
||
pipeline,
|
||
frame,
|
||
shuffled,
|
||
n_splits=5,
|
||
n_repeats=1,
|
||
seed=seed,
|
||
model_name="permuted_logreg",
|
||
verbose=False,
|
||
)
|
||
pooled, _ = result.pooled_summary()
|
||
row = pooled.iloc[0].to_dict()
|
||
row["permutation_seed"] = seed
|
||
rows.append(row)
|
||
out_dir = Path(sys.argv[1]) / "negative_control"
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
table = pd.DataFrame(rows)
|
||
table.to_csv(out_dir / "permutation_logreg.csv", index=False, encoding="utf-8-sig")
|
||
print(table[["permutation_seed", "macro_f1", "accuracy", "top3_accuracy"]].to_string(index=False))
|
||
print(f"[permutation] средний macro-F1: {table['macro_f1'].mean():.4f}")
|
||
PY
|
||
else
|
||
echo
|
||
echo "F9 пропущена (RUN_CONTROLS=${RUN_CONTROLS})."
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сводка по всем запускам и парные сравнения по повторам.
|
||
# ---------------------------------------------------------------------------
|
||
step "Сводка follow-up"
|
||
|
||
PAIRS=(
|
||
"model_rf_default:ref_logreg"
|
||
"model_rf_default:ref_no_lsa"
|
||
"model_rf_default:model_svc_default"
|
||
"rf_grid_best_5:model_rf_default"
|
||
"model_svc_default:ref_logreg"
|
||
"ref_no_lsa:ref_logreg"
|
||
"stylometry_pos_content300:ref_no_lsa"
|
||
"stylometry_pos_pos300:ref_no_lsa"
|
||
"stylometry_pos_lsa_k300:ref_logreg"
|
||
"groups_lean1:ref_logreg"
|
||
"groups_lean2:ref_logreg"
|
||
"groups_lean2_content300:ref_logreg"
|
||
"groups_lean1_rf:model_rf_default"
|
||
"groups_lean2_rf:model_rf_default"
|
||
"min_df2:ref_logreg"
|
||
"char_max10000:ref_logreg"
|
||
"char_ngram_2_4:ref_logreg"
|
||
"char_ngram_3_5:ref_logreg"
|
||
"pos_ngram_1_2:ref_logreg"
|
||
"pos_ngram_1_4:ref_logreg"
|
||
"pos_max1000:ref_logreg"
|
||
"min_df2_rf:model_rf_default"
|
||
"char_max10000_rf:model_rf_default"
|
||
"classweight_logreg_none:ref_logreg"
|
||
"classweight_svc_none:model_svc_default"
|
||
"classweight_rf_none:model_rf_default"
|
||
"classweight_rf_balanced:model_rf_default"
|
||
"final_rf:ref_logreg"
|
||
"final_rf:ref_no_lsa"
|
||
"final_rf:model_rf_default"
|
||
"final_rf:final_rf_nested"
|
||
"control_length_logreg:ref_logreg"
|
||
)
|
||
|
||
run "${PYTHON}" scripts/analyze_followup.py \
|
||
--followup-dir "${FOLLOWUP_DIR}" \
|
||
--pairs "${PAIRS[@]}" \
|
||
--n-boot 1000 --seed "${SEED}"
|
||
|
||
step "Готово"
|
||
echo "Конец follow-up серии: $(date)"
|
||
echo
|
||
echo "Результаты: ${FOLLOWUP_DIR}"
|
||
echo "Сводка: ${FOLLOWUP_DIR}/summary.csv"
|
||
echo "Сравнения: ${FOLLOWUP_DIR}/comparisons.csv"
|
||
echo "Лог: ${LOG_FILE}"
|