Исправлен .gitignore: пакет stylometry/features добавлен в репозиторий

lab1/stylometry/features/__init__.py

@@ -0,0 +1,17 @@
+"""Признаковое пространство стилометрии."""
+
+from .stylometry import (
+    FEATURE_GROUP_PREFIXES,
+    FEATURE_GROUPS,
+    StylometricVectorizer,
+    describe_feature,
+    feature_dictionary,
+)
+
+__all__ = [
+    "FEATURE_GROUP_PREFIXES",
+    "FEATURE_GROUPS",
+    "StylometricVectorizer",
+    "describe_feature",
+    "feature_dictionary",
+]

lab1/stylometry/features/base.py

@@ -0,0 +1,99 @@
+"""Вспомогательные функции для вычисления признаков."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from typing import Iterable, Sequence
+
+import numpy as np
+import pandas as pd
+
+
+def safe_div(numerator: float, denominator: float, default: float = 0.0) -> float:
+    """Деление без исключения при нулевом знаменателе."""
+    if denominator == 0:
+        return default
+    return float(numerator) / float(denominator)
+
+
+def finite(value: float, default: float = 0.0) -> float:
+    """Заменяет NaN и бесконечности на допустимое число."""
+    try:
+        result = float(value)
+    except (TypeError, ValueError):
+        return default
+    return result if math.isfinite(result) else default
+
+
+def as_list(row: pd.Series, column: str) -> list:
+    value = row.get(column, [])
+    if isinstance(value, list):
+        return value
+    if isinstance(value, tuple):
+        return list(value)
+    if value is None or (isinstance(value, float) and math.isnan(value)):
+        return []
+    return list(value)
+
+
+def word_tokens(tokens: Sequence[str]) -> list[str]:
+    """Токены, содержащие хотя бы одну букву или цифру."""
+    return [token for token in tokens if re.search(r"[A-Za-z0-9]", token)]
+
+
+def alphabetic_tokens(tokens: Sequence[str]) -> list[str]:
+    return [token for token in tokens if re.search(r"[A-Za-z]", token)]
+
+
+def lower_tokens(tokens: Sequence[str]) -> list[str]:
+    return [token.lower() for token in tokens]
+
+
+def sentence_word_counts(row: pd.Series) -> list[int]:
+    sentences = as_list(row, "sentences")
+    counts = [len(word_tokens(sentence)) for sentence in sentences]
+    counts = [count for count in counts if count > 0]
+    return counts or [len(word_tokens(as_list(row, "tokens")))]
+
+
+def count_set(tokens: Iterable[str], vocabulary: set[str]) -> int:
+    return sum(1 for token in tokens if token.lower() in vocabulary)
+
+
+def rate_set(tokens: Sequence[str], vocabulary: set[str]) -> float:
+    return safe_div(count_set(tokens, vocabulary), len(tokens))
+
+
+def counter(tokens: Iterable[str]) -> Counter:
+    return Counter(token.lower() for token in tokens)
+
+
+def mean(values: Sequence[float]) -> float:
+    return float(np.mean(values)) if values else 0.0
+
+
+def std(values: Sequence[float]) -> float:
+    return float(np.std(values, ddof=0)) if values else 0.0
+
+
+def median(values: Sequence[float]) -> float:
+    return float(np.median(values)) if values else 0.0
+
+
+def percentile(values: Sequence[float], q: float) -> float:
+    return float(np.percentile(values, q)) if values else 0.0
+
+
+def word_lengths(tokens: Sequence[str]) -> list[int]:
+    lengths: list[int] = []
+    for token in tokens:
+        letters = re.sub(r"[^A-Za-z]", "", token)
+        if letters:
+            lengths.append(len(letters))
+    return lengths
+
+
+def strip_punctuation(token: str) -> str:
+    return re.sub(r"^[^A-Za-z0-9]+|[^A-Za-z0-9]+$", "", token)

lab1/stylometry/features/characters.py

@@ -0,0 +1,38 @@
+"""Символьные признаки: частоты букв, регистр, цифры, пунктуация."""
+
+from __future__ import annotations
+
+import string
+
+import pandas as pd
+
+from .base import safe_div
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    text = str(row.get("text", ""))
+    lower = text.lower()
+    letters = [char for char in lower if "a" <= char <= "z"]
+    n_letters = max(len(letters), 1)
+    n_chars = max(len(text), 1)
+
+    features: dict[str, float] = {}
+    for char in string.ascii_lowercase:
+        features[f"char_freq_{char}"] = safe_div(letters.count(char), n_letters)
+
+    upper = sum(1 for char in text if char.isupper())
+    digits = sum(1 for char in text if char.isdigit())
+    spaces = sum(1 for char in text if char.isspace())
+    punctuation = sum(1 for char in text if not char.isalnum() and not char.isspace())
+    features.update(
+        {
+            "char_letter_count": float(len(letters)),
+            "char_upper_rate": safe_div(upper, n_letters),
+            "char_digit_rate": safe_div(digits, n_chars),
+            "char_space_rate": safe_div(spaces, n_chars),
+            "char_punctuation_rate": safe_div(punctuation, n_chars),
+            "char_apostrophe_rate": safe_div(text.count("'"), n_chars),
+            "char_quote_rate": safe_div(text.count('"'), n_chars),
+        }
+    )
+    return features

lab1/stylometry/features/dialogue.py

@@ -0,0 +1,98 @@
+"""Признаки, специфичные для игровых реплик и разговорной речи."""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from ..lexicons import (
+    CONTRACTIONS,
+    DISCOURSE_MARKERS,
+    HEDGES,
+    INTENSIFIERS,
+    INTERJECTIONS,
+    NEGATIONS,
+    POLITENESS,
+)
+from .base import as_list, safe_div, word_tokens
+
+_LAUGHTER_RE = re.compile(r"\b(?:ha){2,}\b|\bhehe\b|\bhaha\b|\blol\b", re.IGNORECASE)
+_EMOTION_WORDS = {
+    "love", "hate", "like", "want", "feel", "think", "know", "happy",
+    "sad", "angry", "afraid", "scared", "excited", "worried", "glad",
+    "sorry", "hope", "wish", "miss", "enjoy", "fun", "nice", "good",
+    "bad", "terrible", "awful", "great", "wonderful",
+}
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    text = str(row.get("text", ""))
+    tokens = [str(token).lower() for token in as_list(row, "tokens")]
+    words = word_tokens(tokens)
+    n_words = max(len(words), 1)
+    sentences = as_list(row, "sentences")
+    n_sentences = max(len(sentences), 1)
+    alphabetic = [token for token in tokens if re.search(r"[A-Za-z]", token)]
+    n_alpha = max(len(alphabetic), 1)
+    ellipsis = len(re.findall(r"\.{3,}", text))
+    repeated = len(re.findall(r"[!?]{2,}", text))
+
+    return {
+        "dlg_placeholder_rate": safe_div(float(row.get("placeholder_count", 0)), n_words),
+        "dlg_all_caps_rate": safe_div(
+            sum(1 for token in alphabetic if len(token) > 1 and token.isupper()), n_alpha
+        ),
+        "dlg_ellipsis_rate": safe_div(ellipsis, n_sentences),
+        "dlg_repeated_punctuation_rate": safe_div(repeated, n_sentences),
+        "dlg_interjection_rate": safe_div(
+            sum(1 for token in tokens if token in INTERJECTIONS), n_words
+        ),
+        "dlg_laughter_rate": safe_div(len(_LAUGHTER_RE.findall(text)), n_sentences),
+        "dlg_discourse_marker_rate": safe_div(
+            sum(1 for token in tokens if token in DISCOURSE_MARKERS), n_words
+        ),
+        "dlg_hedge_rate": safe_div(
+            sum(1 for token in tokens if token in HEDGES), n_words
+        ),
+        "dlg_intensifier_rate": safe_div(
+            sum(1 for token in tokens if token in INTENSIFIERS), n_words
+        ),
+        "dlg_politeness_rate": safe_div(
+            sum(1 for token in tokens if token in POLITENESS), n_words
+        ),
+        "dlg_negation_rate": safe_div(
+            sum(1 for token in tokens if token in NEGATIONS), n_words
+        ),
+        "dlg_contraction_rate": safe_div(
+            sum(1 for token in tokens if token in CONTRACTIONS or "'" in token), n_words
+        ),
+        "dlg_question_tag_rate": safe_div(
+            len(
+                re.findall(
+                    r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$",
+                    text,
+                    flags=re.IGNORECASE,
+                )
+            ),
+            n_sentences,
+        ),
+        "dlg_filler_rate": safe_div(
+            sum(1 for token in tokens if token in {"um", "uh", "hmm", "mmm", "eh"}), n_words
+        ),
+        "dlg_emotion_word_rate": safe_div(
+            sum(1 for token in tokens if token in _EMOTION_WORDS), n_words
+        ),
+        "dlg_first_person_rate": safe_div(
+            sum(1 for token in tokens if token in {"i", "me", "my", "mine", "myself"}),
+            n_words,
+        ),
+        "dlg_second_person_rate": safe_div(
+            sum(1 for token in tokens if token in {"you", "your", "yours", "yourself"}),
+            n_words,
+        ),
+        "dlg_one_word_sentence_rate": safe_div(
+            sum(1 for sentence in sentences if len(word_tokens(sentence)) == 1),
+            n_sentences,
+        ),
+    }

lab1/stylometry/features/function_words.py

@@ -0,0 +1,46 @@
+"""Относительные частоты функциональных слов.
+
+Это ядро классической стилометрии: закрытые классы слов мало зависят от
+темы, но хорошо отражают авторский или персонажный синтаксис.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from ..lexicons import FUNCTION_WORD_CATEGORIES, FUNCTION_WORDS
+from .base import as_list, safe_div, word_tokens
+
+
+def _safe_name(word: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", word.lower()).strip("_")
+
+
+def _sorted_words() -> list[str]:
+    return sorted(FUNCTION_WORDS)
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = [token.lower() for token in word_tokens(as_list(row, "tokens"))]
+    n_words = max(len(tokens), 1)
+    features: dict[str, float] = {}
+
+    for word in _sorted_words():
+        features[f"fw_word_{_safe_name(word)}"] = safe_div(tokens.count(word), n_words)
+
+    for category, vocabulary in FUNCTION_WORD_CATEGORIES.items():
+        count = sum(1 for token in tokens if token in vocabulary)
+        features[f"fw_cat_{category}"] = safe_div(count, n_words)
+
+    features["fw_total_rate"] = safe_div(
+        sum(1 for token in tokens if token in FUNCTION_WORDS), n_words
+    )
+    features["fw_unique_types"] = float(len(set(tokens) & FUNCTION_WORDS))
+    return features
+
+
+def feature_words() -> list[str]:
+    """Список слов, для которых создаются отдельные признаки."""
+    return _sorted_words()

lab1/stylometry/features/length.py

@@ -0,0 +1,48 @@
+"""Признаки длины и структуры реплики."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from .base import (
+    as_list,
+    finite,
+    mean,
+    median,
+    safe_div,
+    sentence_word_counts,
+    std,
+    word_lengths,
+    word_tokens,
+)
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = as_list(row, "tokens")
+    words = word_tokens(tokens)
+    lengths = word_lengths(words)
+    sentence_lengths = sentence_word_counts(row)
+    n_sentences = max(len(sentence_lengths), 1)
+    text = str(row.get("text", ""))
+
+    return {
+        "len_tokens": float(len(tokens)),
+        "len_words": float(len(words)),
+        "len_chars": float(len(text)),
+        "len_sentences": float(n_sentences),
+        "len_mean_sentence_words": finite(mean(sentence_lengths)),
+        "len_std_sentence_words": finite(std(sentence_lengths)),
+        "len_min_sentence_words": float(min(sentence_lengths)) if sentence_lengths else 0.0,
+        "len_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0,
+        "len_median_sentence_words": finite(median(sentence_lengths)),
+        "len_pct_sentences_lt5": safe_div(sum(1 for value in sentence_lengths if value < 5), n_sentences),
+        "len_pct_sentences_gt20": safe_div(sum(1 for value in sentence_lengths if value > 20), n_sentences),
+        "len_pct_one_word_sentences": safe_div(sum(1 for value in sentence_lengths if value == 1), n_sentences),
+        "len_mean_word_chars": finite(mean(lengths)),
+        "len_std_word_chars": finite(std(lengths)),
+        "len_median_word_chars": finite(median(lengths)),
+        "len_pct_words_ge7": safe_div(sum(1 for value in lengths if value >= 7), len(lengths)),
+        "len_pct_words_ge10": safe_div(sum(1 for value in lengths if value >= 10), len(lengths)),
+        "len_pct_words_ge13": safe_div(sum(1 for value in lengths if value >= 13), len(lengths)),
+        "len_pct_single_letter_words": safe_div(sum(1 for value in lengths if value == 1), len(lengths)),
+    }

lab1/stylometry/features/lexical.py

@@ -0,0 +1,48 @@
+"""Лексические признаки: длины слов и орфографические особенности."""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from .base import as_list, safe_div, word_lengths, word_tokens
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = as_list(row, "tokens")
+    words = word_tokens(tokens)
+    lengths = word_lengths(words)
+    total = max(len(lengths), 1)
+
+    features: dict[str, float] = {}
+    for size in range(1, 10):
+        features[f"lex_word_length_{size}"] = safe_div(
+            sum(1 for value in lengths if value == size), total
+        )
+    features["lex_word_length_10plus"] = safe_div(
+        sum(1 for value in lengths if value >= 10), total
+    )
+
+    alphabetic = [token for token in tokens if re.search(r"[A-Za-z]", token)]
+    n_alpha = max(len(alphabetic), 1)
+    features.update(
+        {
+            "lex_pct_apostrophe_words": safe_div(
+                sum(1 for token in alphabetic if "'" in token), n_alpha
+            ),
+            "lex_pct_capitalized_words": safe_div(
+                sum(1 for token in alphabetic if token[:1].isupper()), n_alpha
+            ),
+            "lex_pct_all_caps_words": safe_div(
+                sum(1 for token in alphabetic if len(token) > 1 and token.isupper()), n_alpha
+            ),
+            "lex_pct_hyphenated_words": safe_div(
+                sum(1 for token in alphabetic if "-" in token), n_alpha
+            ),
+            "lex_pct_digit_words": safe_div(
+                sum(1 for token in alphabetic if any(char.isdigit() for char in token)), n_alpha
+            ),
+        }
+    )
+    return features

lab1/stylometry/features/pos_features.py

@@ -0,0 +1,61 @@
+"""Частеречные признаки на основе NLTK.
+
+Используются две системы тегов: Universal POS (крупные классы) и Penn
+Treebank (более дробные). Дополнительно считаются отношения и отдельные
+грамматические категории.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from ..lexicons import MODALS
+from ..tokenization import ptb_tag_list, upos_tag_list
+from .base import as_list, safe_div, word_tokens
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = as_list(row, "tokens")
+    upos = as_list(row, "upos")
+    ptb = as_list(row, "ptb")
+    lower = [str(token).lower() for token in tokens]
+    n_words = max(len(word_tokens(tokens)), 1)
+
+    features: dict[str, float] = {}
+    for tag in upos_tag_list():
+        features[f"pos_upos_{tag}"] = safe_div(sum(1 for value in upos if value == tag), n_words)
+    for tag in ptb_tag_list():
+        features[f"pos_ptb_{tag}"] = safe_div(sum(1 for value in ptb if value == tag), n_words)
+
+    noun = sum(1 for value in upos if value == "NOUN")
+    verb = sum(1 for value in upos if value == "VERB")
+    adjective = sum(1 for value in upos if value == "ADJ")
+    adverb = sum(1 for value in upos if value == "ADV")
+    pronoun = sum(1 for value in upos if value == "PRON")
+
+    features.update(
+        {
+            "pos_noun_verb_ratio": safe_div(noun + 1.0, verb + 1.0),
+            "pos_adj_adv_ratio": safe_div(adjective + 1.0, adverb + 1.0),
+            "pos_modal_rate": safe_div(sum(1 for token in lower if token in MODALS), n_words),
+            "pos_aux_rate": safe_div(sum(1 for value in upos if value == "AUX"), n_words),
+            "pos_past_tense_rate": safe_div(sum(1 for value in ptb if value == "VBD"), n_words),
+            "pos_present_tense_rate": safe_div(
+                sum(1 for value in ptb if value in {"VBP", "VBZ"}), n_words
+            ),
+            "pos_gerund_rate": safe_div(sum(1 for value in ptb if value == "VBG"), n_words),
+            "pos_participle_rate": safe_div(sum(1 for value in ptb if value == "VBN"), n_words),
+            "pos_comparative_rate": safe_div(
+                sum(1 for value in ptb if value in {"JJR", "RBR"}), n_words
+            ),
+            "pos_superlative_rate": safe_div(
+                sum(1 for value in ptb if value in {"JJS", "RBS"}), n_words
+            ),
+            "pos_pronoun_rate": safe_div(pronoun, n_words),
+            "pos_content_word_rate": safe_div(
+                sum(1 for value in upos if value in {"NOUN", "VERB", "ADJ", "ADV", "NUM"}),
+                n_words,
+            ),
+        }
+    )
+    return features

lab1/stylometry/features/pronouns.py

@@ -0,0 +1,45 @@
+"""Признаки местоимений и референции к говорящему/собеседнику."""
+
+from __future__ import annotations
+
+import math
+
+import pandas as pd
+
+from ..lexicons import PRONOUN_CATEGORIES, WH_WORDS
+from .base import as_list, safe_div, word_tokens
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = [token.lower() for token in word_tokens(as_list(row, "tokens"))]
+    n_words = max(len(tokens), 1)
+    features: dict[str, float] = {}
+
+    for category, vocabulary in PRONOUN_CATEGORIES.items():
+        features[f"pron_{category}_rate"] = safe_div(
+            sum(1 for token in tokens if token in vocabulary), n_words
+        )
+
+    first_singular = sum(
+        1 for token in tokens if token in PRONOUN_CATEGORIES["first_singular"]
+    )
+    second = sum(1 for token in tokens if token in PRONOUN_CATEGORIES["second"])
+    features["pron_i_you_log_ratio"] = math.log((first_singular + 1.0) / (second + 1.0))
+    features["pron_first_person_rate"] = safe_div(
+        first_singular
+        + sum(1 for token in tokens if token in PRONOUN_CATEGORIES["first_plural"]),
+        n_words,
+    )
+    features["pron_third_person_rate"] = safe_div(
+        sum(
+            1
+            for token in tokens
+            if token in PRONOUN_CATEGORIES["third_singular"]
+            or token in PRONOUN_CATEGORIES["third_plural"]
+        ),
+        n_words,
+    )
+    features["pron_wh_rate"] = safe_div(
+        sum(1 for token in tokens if token in WH_WORDS), n_words
+    )
+    return features

lab1/stylometry/features/punctuation.py

@@ -0,0 +1,57 @@
+"""Признаки пунктуации и оформления высказывания."""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from .base import as_list, safe_div, word_tokens
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    text = str(row.get("text", ""))
+    sentences = as_list(row, "sentences")
+    n_sentences = max(len(sentences), 1)
+    n_words = max(len(word_tokens(as_list(row, "tokens"))), 1)
+
+    ellipsis_count = len(re.findall(r"\.{3,}", text))
+    single_periods = len(re.findall(r"(?<!\.)\.(?!\.)", text))
+    exclamations = text.count("!")
+    questions = text.count("?")
+    commas = text.count(",")
+    colons = text.count(":")
+    semicolons = text.count(";")
+    apostrophes = text.count("'")
+    quotes = text.count('"')
+    dashes = text.count("-")
+    parentheses = text.count("(") + text.count(")")
+    multi_exclamation = len(re.findall(r"!{2,}", text))
+    multi_question = len(re.findall(r"\?{2,}", text))
+    mixed = len(re.findall(r"[!?]{2,}", text))
+
+    sentence_texts = [" ".join(sentence) for sentence in sentences if sentence]
+    question_sentences = sum(1 for sentence in sentence_texts if sentence.rstrip().endswith("?"))
+    exclamation_sentences = sum(1 for sentence in sentence_texts if sentence.rstrip().endswith("!"))
+
+    return {
+        "punct_comma_per_sentence": safe_div(commas, n_sentences),
+        "punct_period_per_sentence": safe_div(single_periods, n_sentences),
+        "punct_exclamation_per_sentence": safe_div(exclamations, n_sentences),
+        "punct_question_per_sentence": safe_div(questions, n_sentences),
+        "punct_colon_per_sentence": safe_div(colons, n_sentences),
+        "punct_semicolon_per_sentence": safe_div(semicolons, n_sentences),
+        "punct_ellipsis_per_sentence": safe_div(ellipsis_count, n_sentences),
+        "punct_apostrophe_per_sentence": safe_div(apostrophes, n_sentences),
+        "punct_quote_per_sentence": safe_div(quotes, n_sentences),
+        "punct_dash_per_sentence": safe_div(dashes, n_sentences),
+        "punct_parentheses_per_sentence": safe_div(parentheses, n_sentences),
+        "punct_multi_exclamation_per_sentence": safe_div(multi_exclamation, n_sentences),
+        "punct_multi_question_per_sentence": safe_div(multi_question, n_sentences),
+        "punct_mixed_per_sentence": safe_div(mixed, n_sentences),
+        "punct_question_sentence_rate": safe_div(question_sentences, n_sentences),
+        "punct_exclamation_sentence_rate": safe_div(exclamation_sentences, n_sentences),
+        "punct_comma_per_token": safe_div(commas, n_words),
+        "punct_ellipsis_per_token": safe_div(ellipsis_count, n_words),
+        "punct_apostrophe_per_token": safe_div(apostrophes, n_words),
+    }

lab1/stylometry/features/readability.py

@@ -0,0 +1,66 @@
+"""Метрики читаемости.
+
+Формулы адаптированы для коротких разговорных реплик, поэтому значения
+следует рассматривать как относительные признаки, а не как точные оценки
+сложности текста. Число слогов вычисляется эвристически.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from .base import as_list, finite, safe_div, sentence_word_counts, word_tokens
+
+
+def count_syllables(word: str) -> int:
+    """Эвристическая оценка числа слогов в английском слове."""
+    word = re.sub(r"[^a-z]", "", word.lower())
+    if not word:
+        return 0
+    if len(word) <= 3:
+        return 1
+    if word.endswith("e") and not word.endswith("le"):
+        word = word[:-1]
+    groups = re.findall(r"[aeiouy]+", word)
+    return max(len(groups), 1)
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = word_tokens(as_list(row, "tokens"))
+    words = [re.sub(r"[^A-Za-z]", "", token).lower() for token in tokens]
+    words = [word for word in words if word]
+    n_words = max(len(words), 1)
+    sentence_lengths = sentence_word_counts(row)
+    n_sentences = max(len(sentence_lengths), 1)
+    syllables = [count_syllables(word) for word in words]
+    n_syllables = sum(syllables)
+    polysyllables = sum(1 for value in syllables if value >= 3)
+    n_letters = sum(len(word) for word in words)
+
+    words_per_sentence = safe_div(n_words, n_sentences)
+    syllables_per_word = safe_div(n_syllables, n_words)
+    polysyllable_rate = safe_div(polysyllables, n_words)
+    letters_per_word = safe_div(n_letters, n_words)
+
+    flesch = 206.835 - 1.015 * words_per_sentence - 84.6 * syllables_per_word
+    fk_grade = 0.39 * words_per_sentence + 11.8 * syllables_per_word - 15.59
+    gunning_fog = 0.4 * (words_per_sentence + 100.0 * polysyllable_rate)
+    smog = 1.0430 * (30.0 * safe_div(polysyllables, n_sentences)) ** 0.5 + 3.1291
+    coleman_liau = 0.0588 * (letters_per_word * 100.0) - 0.296 * (
+        safe_div(n_sentences, n_words) * 100.0
+    ) - 15.8
+    ari = 4.71 * letters_per_word + 0.5 * words_per_sentence - 21.43
+
+    return {
+        "read_syllables_per_word": finite(syllables_per_word),
+        "read_polysyllable_rate": finite(polysyllable_rate),
+        "read_flesch": finite(flesch),
+        "read_fk_grade": finite(fk_grade),
+        "read_gunning_fog": finite(gunning_fog),
+        "read_smog": finite(smog),
+        "read_coleman_liau": finite(coleman_liau),
+        "read_ari": finite(ari),
+        "read_words_per_sentence": finite(words_per_sentence),
+    }

lab1/stylometry/features/stylometry.py

@@ -0,0 +1,192 @@
+"""Агрегатор ручных стилометрических признаков.
+
+Класс не обучается на данных: все формулы фиксированы. Поэтому его можно
+безопасно применять ко всей таблице до кросс-валидации. Обучаемые
+преобразования (TF-IDF, SVD, масштабирование) вынесены в другие классы и
+настраиваются только на обучающих блоках.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Iterable, Sequence
+
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+from . import (
+    characters,
+    dialogue,
+    function_words,
+    length,
+    lexical,
+    pos_features,
+    pronouns,
+    punctuation,
+    readability,
+    syntax,
+    vocabulary,
+)
+
+FEATURE_GROUPS: dict[str, Callable[[pd.Series], dict[str, float]]] = {
+    "length": length.extract,
+    "lexical": lexical.extract,
+    "vocabulary": vocabulary.extract,
+    "characters": characters.extract,
+    "punctuation": punctuation.extract,
+    "function_words": function_words.extract,
+    "pronouns": pronouns.extract,
+    "pos": pos_features.extract,
+    "syntax": syntax.extract,
+    "readability": readability.extract,
+    "dialogue": dialogue.extract,
+}
+
+FEATURE_GROUP_PREFIXES: dict[str, str] = {
+    "length": "len_",
+    "lexical": "lex_",
+    "vocabulary": "vocab_",
+    "characters": "char_",
+    "punctuation": "punct_",
+    "function_words": "fw_",
+    "pronouns": "pron_",
+    "pos": "pos_",
+    "syntax": "syn_",
+    "readability": "read_",
+    "dialogue": "dlg_",
+}
+
+EXPLORATORY_GROUPS = {"vocabulary", "readability"}
+
+
+def default_groups() -> list[str]:
+    return list(FEATURE_GROUPS)
+
+
+def _validate_groups(groups: Sequence[str]) -> list[str]:
+    unknown = [group for group in groups if group not in FEATURE_GROUPS]
+    if unknown:
+        raise ValueError(
+            f"Неизвестные группы признаков: {unknown}. "
+            f"Доступны: {sorted(FEATURE_GROUPS)}"
+        )
+    return list(groups)
+
+
+class StylometricVectorizer(BaseEstimator, TransformerMixin):
+    """Считает матрицу ручных признаков по таблице реплик.
+
+    Ожидает DataFrame с колонками ``tokens``, ``upos``, ``ptb``, ``sentences``,
+    ``text`` и ``placeholder_count``. Такие колонки создаёт
+    :class:`stylometry.tokenization.Tokenizer`.
+    """
+
+    def __init__(self, groups: Sequence[str] | None = None) -> None:
+        self.groups = groups
+
+    def _selected_groups(self) -> list[str]:
+        groups = self.groups if self.groups is not None else default_groups()
+        return _validate_groups(groups)
+
+    @staticmethod
+    def _extract_row(row: pd.Series, groups: Sequence[str]) -> dict[str, float]:
+        features: dict[str, float] = {}
+        for group in groups:
+            features.update(FEATURE_GROUPS[group](row))
+        return features
+
+    def fit(self, X: pd.DataFrame, y=None) -> "StylometricVectorizer":
+        if not isinstance(X, pd.DataFrame):
+            raise TypeError("StylometricVectorizer ожидает pandas.DataFrame.")
+        if len(X) == 0:
+            raise ValueError("Пустая таблица признаков.")
+        groups = self._selected_groups()
+        sample = self._extract_row(X.iloc[0], groups)
+        self.feature_names_in_ = np.array(list(sample.keys()), dtype=object)
+        self.n_features_in_ = len(self.feature_names_in_)
+        return self
+
+    def transform(self, X: pd.DataFrame) -> np.ndarray:
+        if not isinstance(X, pd.DataFrame):
+            raise TypeError("StylometricVectorizer ожидает pandas.DataFrame.")
+        groups = self._selected_groups()
+        rows: list[list[float]] = []
+        names: list[str] | None = None
+        for _, row in X.iterrows():
+            features = self._extract_row(row, groups)
+            if names is None:
+                names = list(features.keys())
+            rows.append([float(features.get(name, 0.0)) for name in names])
+        if names is None:
+            raise ValueError("Не удалось вычислить признаки: таблица пуста.")
+        self.feature_names_in_ = np.array(names, dtype=object)
+        self.n_features_in_ = len(names)
+        return np.asarray(rows, dtype=float)
+
+    def get_feature_names_out(self, input_features=None) -> np.ndarray:
+        if not hasattr(self, "feature_names_in_"):
+            raise RuntimeError("Сначала вызовите fit или transform.")
+        return self.feature_names_in_
+
+
+def feature_group(feature_name: str) -> str:
+    """Определяет группу признака по его префиксу."""
+    for group, prefix in FEATURE_GROUP_PREFIXES.items():
+        if feature_name.startswith(prefix):
+            return group
+    return "other"
+
+
+def describe_feature(feature_name: str) -> str:
+    """Возвращает краткое описание признака для словаря и отчёта."""
+    group = feature_group(feature_name)
+    if group == "function_words":
+        if feature_name.startswith("fw_word_"):
+            word = feature_name.removeprefix("fw_word_").replace("_", " ")
+            return f"Относительная частота функционального слова «{word}»."
+        if feature_name.startswith("fw_cat_"):
+            category = feature_name.removeprefix("fw_cat_")
+            return f"Относительная частота категории функциональных слов «{category}»."
+        return "Суммарная относительная частота функциональных слов."
+    if group == "characters" and feature_name.startswith("char_freq_"):
+        letter = feature_name.removeprefix("char_freq_")
+        return f"Относительная частота буквы «{letter}» среди латинских букв."
+    if group == "pos" and feature_name.startswith("pos_upos_"):
+        tag = feature_name.removeprefix("pos_upos_")
+        return f"Доля универсального частеречного тега {tag}."
+    if group == "pos" and feature_name.startswith("pos_ptb_"):
+        tag = feature_name.removeprefix("pos_ptb_")
+        return f"Доля тега Penn Treebank {tag}."
+    if group == "lexical" and feature_name.startswith("lex_word_length_"):
+        size = feature_name.removeprefix("lex_word_length_")
+        return f"Доля слов длины {size} (в символах)."
+    descriptions = {
+        "length": "Признак длины или структуры реплики.",
+        "lexical": "Лексический признак длины слова или орфографии.",
+        "vocabulary": "Признак словарного богатства; нестабилен на коротких текстах.",
+        "characters": "Символьный признак: регистр, цифры, пунктуация, буквы.",
+        "punctuation": "Признак пунктуационного оформления.",
+        "function_words": "Признак функциональных слов.",
+        "pronouns": "Признак местоимений и референции.",
+        "pos": "Частеречный признак.",
+        "syntax": "Синтаксический признак-прокси.",
+        "readability": "Метрика читаемости; на коротких репликах приблизительна.",
+        "dialogue": "Признак разговорной речи и игровых реплик.",
+    }
+    return descriptions.get(group, "Стилометрический признак.")
+
+
+def feature_dictionary(feature_names: Iterable[str]) -> pd.DataFrame:
+    """Строит таблицу с описанием признаков для отчёта."""
+    rows = []
+    for name in feature_names:
+        group = feature_group(name)
+        rows.append(
+            {
+                "feature": name,
+                "group": group,
+                "description": describe_feature(name),
+                "status": "exploratory" if group in EXPLORATORY_GROUPS else "core",
+            }
+        )
+    return pd.DataFrame(rows)

lab1/stylometry/features/syntax.py

@@ -0,0 +1,112 @@
+"""Синтаксические признаки-прокси.
+
+Полноценный синтаксический анализ не используется: для коротких реплик он
+нестабилен и требует тяжёлых моделей. Вместо этого считаются устойчивые
+приближения по функциям слов, длине предложений и окончаниям.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from ..lexicons import COORDINATORS, MODALS, NEGATIONS, SUBORDINATORS
+from .base import as_list, safe_div, sentence_word_counts, word_tokens
+
+_IMPERATIVE_VERBS = {
+    "go", "come", "look", "take", "get", "give", "tell", "let", "stop",
+    "wait", "listen", "try", "keep", "put", "make", "do", "be", "have",
+    "see", "say", "help", "leave", "stay", "bring", "show", "find",
+}
+
+
+def _sentence_first_words(row: pd.Series) -> list[str]:
+    sentences = as_list(row, "sentences")
+    first_words: list[str] = []
+    for sentence in sentences:
+        for token in sentence:
+            if re.search(r"[A-Za-z]", token):
+                first_words.append(token.lower())
+                break
+    return first_words
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = [str(token).lower() for token in as_list(row, "tokens")]
+    words = word_tokens(tokens)
+    n_words = max(len(words), 1)
+    sentences = as_list(row, "sentences")
+    n_sentences = max(len(sentences), 1)
+    sentence_lengths = sentence_word_counts(row)
+    first_words = _sentence_first_words(row)
+    text = str(row.get("text", ""))
+
+    coordinators = sum(1 for token in tokens if token in COORDINATORS)
+    subordinators = sum(1 for token in tokens if token in SUBORDINATORS)
+    clauses = coordinators + subordinators
+
+    passive = len(
+        re.findall(
+            r"\b(?:be|is|are|was|were|been|being)\s+"
+            r"(?:\w+ed|known|made|given|taken|seen|done|written|spoken)\b",
+            text,
+            flags=re.IGNORECASE,
+        )
+    )
+    question_tags = len(
+        re.findall(
+            r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$",
+            text,
+            flags=re.IGNORECASE,
+        )
+    )
+    imperative = sum(1 for word in first_words if word in _IMPERATIVE_VERBS)
+    sentence_initial_conjunction = sum(
+        1 for word in first_words if word in COORDINATORS or word in SUBORDINATORS
+    )
+
+    return {
+        "syn_mean_clauses_per_sentence": safe_div(clauses, n_sentences),
+        "syn_coordinator_per_sentence": safe_div(coordinators, n_sentences),
+        "syn_subordinator_per_sentence": safe_div(subordinators, n_sentences),
+        "syn_sentence_initial_conjunction_rate": safe_div(
+            sentence_initial_conjunction, n_sentences
+        ),
+        "syn_question_rate": safe_div(
+            sum(
+                1
+                for sentence in sentences
+                if sentence and " ".join(sentence).rstrip().endswith("?")
+            ),
+            n_sentences,
+        ),
+        "syn_exclamation_rate": safe_div(
+            sum(
+                1
+                for sentence in sentences
+                if sentence and " ".join(sentence).rstrip().endswith("!")
+            ),
+            n_sentences,
+        ),
+        "syn_modal_rate": safe_div(sum(1 for token in tokens if token in MODALS), n_words),
+        "syn_negation_per_sentence": safe_div(
+            sum(1 for token in tokens if token in NEGATIONS), n_sentences
+        ),
+        "syn_that_rate": safe_div(tokens.count("that"), n_words),
+        "syn_to_rate": safe_div(tokens.count("to"), n_words),
+        "syn_ing_rate": safe_div(
+            sum(1 for token in words if token.endswith("ing")), n_words
+        ),
+        "syn_ed_rate": safe_div(
+            sum(1 for token in words if token.endswith("ed")), n_words
+        ),
+        "syn_ly_rate": safe_div(
+            sum(1 for token in words if token.endswith("ly")), n_words
+        ),
+        "syn_passive_proxy_per_sentence": safe_div(passive, n_sentences),
+        "syn_question_tag_per_sentence": safe_div(question_tags, n_sentences),
+        "syn_imperative_proxy_rate": safe_div(imperative, n_sentences),
+        "syn_mean_sentence_words": safe_div(sum(sentence_lengths), n_sentences),
+        "syn_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0,
+    }

lab1/stylometry/features/vocabulary.py

@@ -0,0 +1,67 @@
+"""Признаки словарного богатства.
+
+На коротких репликах эти признаки нестабильны. Они включены как
+дополнительная группа и должны проверяться отдельно в ablation-эксперименте.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+
+import pandas as pd
+
+from .base import as_list, finite, safe_div, word_tokens
+
+
+def _yule_k(counter: Counter[str], n_tokens: int) -> float:
+    if n_tokens <= 1:
+        return 0.0
+    frequency_of_frequencies = Counter(counter.values())
+    second_moment = sum(
+        frequency * frequency * count
+        for frequency, count in frequency_of_frequencies.items()
+    )
+    return finite(10_000.0 * (second_moment - n_tokens) / (n_tokens * n_tokens))
+
+
+def _simpson_d(counter: Counter[str], n_tokens: int) -> float:
+    if n_tokens <= 1:
+        return 0.0
+    numerator = sum(count * (count - 1) for count in counter.values())
+    return finite(numerator / (n_tokens * (n_tokens - 1)))
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = [token.lower() for token in word_tokens(as_list(row, "tokens"))]
+    counter = Counter(tokens)
+    n_tokens = len(tokens)
+    n_types = len(counter)
+    hapax = sum(1 for count in counter.values() if count == 1)
+    dis = sum(1 for count in counter.values() if count == 2)
+
+    ttr = safe_div(n_types, n_tokens)
+    guiraud = safe_div(n_types, math.sqrt(n_tokens))
+    herdan = finite(math.log(n_types) / math.log(n_tokens)) if n_tokens > 1 and n_types > 1 else 0.0
+    hapax_ratio = safe_div(hapax, n_types)
+    dis_ratio = safe_div(dis, n_types)
+    if n_types > 0 and hapax_ratio < 1.0:
+        honore = finite(100.0 * math.log(n_tokens) / (1.0 - hapax_ratio))
+    else:
+        honore = 0.0
+
+    top10 = sum(count for _, count in counter.most_common(10))
+    return {
+        "vocab_tokens": float(n_tokens),
+        "vocab_types": float(n_types),
+        "vocab_ttr": ttr,
+        "vocab_guiraud": finite(guiraud),
+        "vocab_herdan": herdan,
+        "vocab_yule_k": _yule_k(counter, n_tokens),
+        "vocab_simpson_d": _simpson_d(counter, n_tokens),
+        "vocab_hapax_ratio": hapax_ratio,
+        "vocab_dis_ratio": dis_ratio,
+        "vocab_honore_r": honore,
+        "vocab_top10_coverage": safe_div(top10, n_tokens),
+        "vocab_mean_word_frequency": safe_div(n_tokens, n_types),
+    }