Первичная версия: стилометрия реплик, LSA и протокол экспериментов

lab1/tests/test_cleaning.py

@@ -0,0 +1,33 @@
+"""Тесты нормализации текста."""
+
+from __future__ import annotations
+
+from stylometry.cleaning import (
+    clean_text,
+    count_placeholders,
+    normalize_placeholders,
+    normalize_unicode,
+    normalized_key,
+)
+
+
+def test_placeholders_are_replaced() -> None:
+    text = "Hello, <PLAYER>! Your <FARM> is nice."
+    assert normalize_placeholders(text) == "Hello, PLAYER! Your FARM is nice."
+    assert count_placeholders(text) == 2
+
+
+def test_unknown_placeholder_becomes_entity() -> None:
+    assert normalize_placeholders("Look, <UNKNOWN>!") == "Look, ENTITY!"
+
+
+def test_unicode_normalization() -> None:
+    assert normalize_unicode("Oh… really—yes") == "Oh... really-yes"
+
+
+def test_whitespace_collapse() -> None:
+    assert clean_text("  Hello\n\n  world  ") == "Hello world"
+
+
+def test_normalized_key_ignores_case_and_punctuation() -> None:
+    assert normalized_key("Oh, <PLAYER>!") == normalized_key("oh player")

lab1/tests/test_cv.py

@@ -0,0 +1,35 @@
+"""Тесты кросс-валидации на синтетических данных."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from stylometry.cv import run_repeated_cv
+
+
+def test_repeated_cv_runs_and_has_all_predictions() -> None:
+    rng = np.random.default_rng(0)
+    n_samples = 60
+    n_classes = 3
+    X = pd.DataFrame(
+        {
+            "feature_1": rng.normal(size=n_samples),
+            "feature_2": rng.normal(size=n_samples),
+        }
+    )
+    y = np.repeat([f"c{index}" for index in range(n_classes)], 20)
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.pipeline import Pipeline
+    from sklearn.preprocessing import StandardScaler
+
+    pipeline = Pipeline(
+        [("scale", StandardScaler()), ("model", LogisticRegression(max_iter=1000))]
+    )
+    result = run_repeated_cv(
+        pipeline, X, y, n_splits=3, n_repeats=2, seed=42, verbose=False
+    )
+    assert len(result.oof) == n_samples * 2
+    assert set(result.oof["y_true"]) == set(y)
+    assert result.fold_metrics["macro_f1"].between(0, 1).all()
+    assert result.labels.tolist() == ["c0", "c1", "c2"]

lab1/tests/test_delta.py

@@ -0,0 +1,41 @@
+"""Тесты дельты Барроуза."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from stylometry.delta import DeltaClassifier, MFWVectorizer
+
+
+def _toy_frame() -> pd.DataFrame:
+    texts = [
+        "the cat sat on the mat",
+        "the cat sat on the mat again",
+        "the dog sat on the log",
+        "the dog sat on the log again",
+        "a cat and a dog",
+        "a cat and a dog again",
+    ]
+    labels = ["cat", "cat", "dog", "dog", "cat", "dog"]
+    return pd.DataFrame({"text": texts, "character": labels})
+
+
+def test_mfw_vectorizer_shape() -> None:
+    frame = _toy_frame()
+    vectorizer = MFWVectorizer(n_features=5)
+    matrix = vectorizer.fit_transform(frame)
+    assert matrix.shape == (6, 5)
+    assert np.isfinite(matrix).all()
+
+
+def test_delta_fit_predict() -> None:
+    frame = _toy_frame()
+    classifier = DeltaClassifier(n_features=5, metric="manhattan")
+    classifier.fit(frame, frame["character"].values)
+    predictions = classifier.predict(frame)
+    assert len(predictions) == len(frame)
+    assert set(predictions) <= {"cat", "dog"}
+    probabilities = classifier.predict_proba(frame)
+    assert probabilities.shape == (len(frame), 2)
+    assert np.allclose(probabilities.sum(axis=1), 1.0)

lab1/tests/test_features.py

@@ -0,0 +1,38 @@
+"""Тесты ручных стилометрических признаков."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from stylometry.datasets import load_feature_frame
+from stylometry.features import StylometricVectorizer, feature_dictionary
+
+
+def test_feature_matrix_shape_and_finiteness() -> None:
+    frame = load_feature_frame(verbose=False).head(20)
+    vectorizer = StylometricVectorizer()
+    matrix = vectorizer.fit_transform(frame)
+    names = vectorizer.get_feature_names_out()
+    assert matrix.shape == (20, len(names))
+    assert np.isfinite(matrix).all()
+    assert len(set(names)) == len(names)
+
+
+def test_feature_dictionary() -> None:
+    frame = load_feature_frame(verbose=False).head(2)
+    vectorizer = StylometricVectorizer()
+    vectorizer.fit(frame)
+    dictionary = feature_dictionary(vectorizer.get_feature_names_out())
+    assert len(dictionary) == len(vectorizer.get_feature_names_out())
+    assert {"feature", "group", "description", "status"} <= set(dictionary.columns)
+
+
+def test_group_subset() -> None:
+    frame = load_feature_frame(verbose=False).head(5)
+    vectorizer = StylometricVectorizer(groups=["length", "punctuation"])
+    matrix = vectorizer.fit_transform(frame)
+    assert matrix.shape[0] == 5
+    assert all(
+        name.startswith("len_") or name.startswith("punct_")
+        for name in vectorizer.get_feature_names_out()
+    )

lab1/tests/test_io.py

@@ -0,0 +1,52 @@
+"""Тесты чтения и валидации корпуса."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+from stylometry.io import (
+    DEFAULT_DATA_PATH,
+    compare_with_stats,
+    corpus_fingerprint,
+    read_corpus,
+    read_stats,
+    validate_corpus,
+)
+
+
+def test_read_corpus_shape() -> None:
+    frame = read_corpus()
+    assert len(frame) == 2005
+    assert frame["character"].nunique() == 34
+    assert {"line_id", "text", "character", "text_raw", "placeholder_count"} <= set(frame.columns)
+    assert frame["text"].str.len().gt(0).all()
+
+
+def test_corpus_is_cp1251() -> None:
+    raw = DEFAULT_DATA_PATH.read_bytes()
+    assert raw.decode("cp1251").startswith("Реплика;Персонаж")
+
+
+def test_validate_corpus() -> None:
+    frame = read_corpus()
+    stats = validate_corpus(frame)
+    assert stats["n_lines"] == 2005
+    assert stats["n_characters"] == 34
+    assert stats["n_duplicate_texts"] == 0
+    assert stats["n_words"] > 30_000
+
+
+def test_compare_with_stats() -> None:
+    frame = read_corpus()
+    comparison = compare_with_stats(frame, read_stats())
+    assert not comparison.empty
+    assert (comparison["difference"] == 0).all()
+
+
+def test_fingerprint_is_stable() -> None:
+    first = corpus_fingerprint()
+    second = corpus_fingerprint()
+    assert first == second
+    assert len(first) == 64

lab1/tests/test_lsa.py

@@ -0,0 +1,45 @@
+"""Тесты TF-IDF и LSA."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from stylometry.datasets import load_feature_frame
+from stylometry.lsa import (
+    make_char_ngram_vectorizer,
+    make_lsa_vectorizer,
+    make_pos_ngram_vectorizer,
+    make_text_vectorizer,
+)
+
+
+def test_lsa_word_shape_and_variance() -> None:
+    frame = load_feature_frame(verbose=False).head(100)
+    vectorizer = make_lsa_vectorizer("word", n_components=10)
+    matrix = vectorizer.fit_transform(frame)
+    assert matrix.shape == (100, 10)
+    assert np.isfinite(matrix).all()
+    assert 0 <= vectorizer.cumulative_explained_variance_[-1] <= 1.0
+    assert len(vectorizer.top_terms(5)) == 10
+
+
+def test_lsa_pos_and_char() -> None:
+    frame = load_feature_frame(verbose=False).head(100)
+    for variant in ("pos", "char"):
+        vectorizer = make_lsa_vectorizer(variant, n_components=5)
+        matrix = vectorizer.fit_transform(frame)
+        assert matrix.shape[0] == 100
+        assert matrix.shape[1] == 5
+        assert np.isfinite(matrix).all()
+
+
+def test_text_vectorizers() -> None:
+    frame = load_feature_frame(verbose=False).head(50)
+    for vectorizer in (
+        make_text_vectorizer("word"),
+        make_pos_ngram_vectorizer(),
+        make_char_ngram_vectorizer(),
+    ):
+        matrix = vectorizer.fit_transform(frame)
+        assert matrix.shape[0] == 50
+        assert matrix.shape[1] > 0

lab1/tests/test_metadata.py

@@ -0,0 +1,36 @@
+"""Тесты анализа по метаданным персонажей."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from stylometry.metadata import (
+    group_confusions,
+    load_character_metadata,
+    subgroup_report,
+)
+
+
+def test_metadata_table() -> None:
+    metadata = load_character_metadata()
+    assert len(metadata) == 34
+    assert {"character", "gender", "age_group", "species", "role"} <= set(metadata.columns)
+
+
+def test_subgroup_report() -> None:
+    metadata = load_character_metadata()
+    y_true = ["Abigail", "Abigail", "Alex", "Alex"]
+    y_pred = ["Abigail", "Alex", "Alex", "Alex"]
+    labels = ["Abigail", "Alex"]
+    report = subgroup_report(y_true, y_pred, labels, metadata, "gender")
+    assert {"group", "weighted_f1", "n_samples"} <= set(report.columns)
+    assert report["n_samples"].sum() == 4
+
+
+def test_group_confusions() -> None:
+    metadata = load_character_metadata()
+    y_true = ["Abigail", "Abigail", "Alex", "Alex"]
+    y_pred = ["Abigail", "Alex", "Alex", "Abigail"]
+    table = group_confusions(y_true, y_pred, metadata, "gender")
+    assert not table.empty
+    assert {"true_group", "pred_group", "count"} <= set(table.columns)

lab1/tests/test_metrics.py

@@ -0,0 +1,72 @@
+"""Тесты метрик и статистических критериев."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from stylometry.evaluation import (
+    bootstrap_ci,
+    classification_metrics,
+    confusion_matrix_frame,
+    per_class_report,
+)
+from stylometry.stats import (
+    bootstrap_difference,
+    holm_bonferroni,
+    mcnemar_exact,
+    wilcoxon_signed_rank,
+)
+
+
+def test_classification_metrics_perfect() -> None:
+    y_true = np.array(["a", "b", "c", "a", "b", "c"])
+    y_pred = y_true.copy()
+    metrics = classification_metrics(y_true, y_pred)
+    assert metrics["accuracy"] == 1.0
+    assert metrics["macro_f1"] == 1.0
+    assert metrics["balanced_accuracy"] == 1.0
+
+
+def test_confusion_matrix_and_report() -> None:
+    y_true = np.array(["a", "a", "b", "b"])
+    y_pred = np.array(["a", "b", "b", "b"])
+    matrix = confusion_matrix_frame(y_true, y_pred, ["a", "b"])
+    assert matrix.loc["a", "a"] == 1
+    report = per_class_report(y_true, y_pred, ["a", "b"])
+    assert {"precision", "recall", "f1", "support"} <= set(report.columns)
+
+
+def test_bootstrap_ci() -> None:
+    y_true = np.array(["a", "b", "a", "b", "a", "b"])
+    y_pred = np.array(["a", "b", "a", "a", "a", "b"])
+    interval = bootstrap_ci(y_true, y_pred, ["a", "b"], n_boot=50, seed=1)
+    assert interval["lower"] <= interval["upper"]
+
+
+def test_mcnemar() -> None:
+    y_true = np.array([1, 1, 0, 0, 1, 0])
+    pred_a = np.array([1, 1, 0, 0, 1, 0])
+    pred_b = np.array([1, 0, 0, 1, 1, 0])
+    result = mcnemar_exact(y_true, pred_a, pred_b)
+    assert result["b"] >= 0 and result["c"] >= 0
+    assert 0 <= result["p_value"] <= 1
+
+
+def test_holm() -> None:
+    adjusted = holm_bonferroni([0.01, 0.02, 0.03])
+    assert len(adjusted) == 3
+    assert all(0 <= value <= 1 for value in adjusted)
+
+
+def test_wilcoxon() -> None:
+    result = wilcoxon_signed_rank([0.5, 0.6, 0.7], [0.4, 0.6, 0.5])
+    assert 0 <= result["p_value"] <= 1
+
+
+def test_bootstrap_difference() -> None:
+    y_true = np.array([0, 1, 2, 0, 1, 2])
+    pred_a = np.array([0, 1, 2, 0, 1, 2])
+    pred_b = np.array([0, 1, 1, 0, 1, 1])
+    result = bootstrap_difference(y_true, pred_a, pred_b, labels=[0, 1, 2], n_boot=50)
+    assert result["lower"] <= result["upper"]

lab1/tests/test_tokenization.py

@@ -0,0 +1,40 @@
+"""Тесты NLTK-токенизации и частеречной разметки."""
+
+from __future__ import annotations
+
+from stylometry.tokenization import Tokenizer, penn_to_upos
+
+
+def test_penn_to_upos() -> None:
+    assert penn_to_upos("the", "DT") == "DET"
+    assert penn_to_upos("run", "VB") == "VERB"
+    assert penn_to_upos("is", "VBZ") == "AUX"
+    assert penn_to_upos("because", "IN") == "SCONJ"
+    assert penn_to_upos("quickly", "RB") == "ADV"
+
+
+def test_analyze_line() -> None:
+    tokenizer = Tokenizer(use_nltk=True)
+    analyzed = tokenizer.analyze("Oh... hi. I don't know, Mr. Qi.")
+    assert analyzed.tokens
+    assert len(analyzed.tokens) == len(analyzed.upos) == len(analyzed.ptb)
+    assert analyzed.sentences
+    assert analyzed.pos_text
+    assert analyzed.fw_text
+    assert analyzed.content_text
+
+
+def test_placeholders_are_tagged_as_words() -> None:
+    tokenizer = Tokenizer(use_nltk=True)
+    analyzed = tokenizer.analyze("Hello, PLAYER! You are nice.")
+    assert "PLAYER" in analyzed.tokens
+    assert len(analyzed.upos) == len(analyzed.tokens)
+
+
+def test_fallback_regex_tokenizer() -> None:
+    tokenizer = Tokenizer(use_nltk=False)
+    analyzed = tokenizer.analyze("Hello, world! It's fine.")
+    assert analyzed.tokens == ["Hello", ",", "world", "!", "It's", "fine", "."]
+    assert len(analyzed.upos) == len(analyzed.tokens)
+    assert analyzed.upos[0] == "X"
+    assert "PUNCT" in analyzed.upos