From 10e2444d2342e2d02c94a06de884ad7bbee83bca Mon Sep 17 00:00:00 2001 From: Vladislav Kopnin Date: Sun, 25 Oct 2026 15:52:25 +0300 Subject: [PATCH] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20.gitignore:=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82=20s?= =?UTF-8?q?tylometry/features=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20=D0=B2=20=D1=80=D0=B5=D0=BF=D0=BE=D0=B7=D0=B8=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lab1/.gitignore | 13 +- lab1/stylometry/features/__init__.py | 17 ++ lab1/stylometry/features/base.py | 99 +++++++++++ lab1/stylometry/features/characters.py | 38 ++++ lab1/stylometry/features/dialogue.py | 98 +++++++++++ lab1/stylometry/features/function_words.py | 46 +++++ lab1/stylometry/features/length.py | 48 ++++++ lab1/stylometry/features/lexical.py | 48 ++++++ lab1/stylometry/features/pos_features.py | 61 +++++++ lab1/stylometry/features/pronouns.py | 45 +++++ lab1/stylometry/features/punctuation.py | 57 ++++++ lab1/stylometry/features/readability.py | 66 +++++++ lab1/stylometry/features/stylometry.py | 192 +++++++++++++++++++++ lab1/stylometry/features/syntax.py | 112 ++++++++++++ lab1/stylometry/features/vocabulary.py | 67 +++++++ 15 files changed, 1000 insertions(+), 7 deletions(-) create mode 100644 lab1/stylometry/features/__init__.py create mode 100644 lab1/stylometry/features/base.py create mode 100644 lab1/stylometry/features/characters.py create mode 100644 lab1/stylometry/features/dialogue.py create mode 100644 lab1/stylometry/features/function_words.py create mode 100644 lab1/stylometry/features/length.py create mode 100644 lab1/stylometry/features/lexical.py create mode 100644 lab1/stylometry/features/pos_features.py create mode 100644 lab1/stylometry/features/pronouns.py create mode 100644 lab1/stylometry/features/punctuation.py create mode 100644 lab1/stylometry/features/readability.py create mode 100644 lab1/stylometry/features/stylometry.py create mode 100644 lab1/stylometry/features/syntax.py create mode 100644 lab1/stylometry/features/vocabulary.py diff --git a/lab1/.gitignore b/lab1/.gitignore index 5c8e83e..4d32bfb 100644 --- a/lab1/.gitignore +++ b/lab1/.gitignore @@ -2,10 +2,9 @@ __pycache__/ *.py[cod] .pytest_cache/ .idea/ -nltk_data/ -data/processed/ -features/ -results/* -!results/README.md - -exports/ +/nltk_data/ +/data/processed/ +/features/ +/results/* +!/results/README.md +/exports/ diff --git a/lab1/stylometry/features/__init__.py b/lab1/stylometry/features/__init__.py new file mode 100644 index 0000000..b9b871c --- /dev/null +++ b/lab1/stylometry/features/__init__.py @@ -0,0 +1,17 @@ +"""Признаковое пространство стилометрии.""" + +from .stylometry import ( + FEATURE_GROUP_PREFIXES, + FEATURE_GROUPS, + StylometricVectorizer, + describe_feature, + feature_dictionary, +) + +__all__ = [ + "FEATURE_GROUP_PREFIXES", + "FEATURE_GROUPS", + "StylometricVectorizer", + "describe_feature", + "feature_dictionary", +] diff --git a/lab1/stylometry/features/base.py b/lab1/stylometry/features/base.py new file mode 100644 index 0000000..4a0808c --- /dev/null +++ b/lab1/stylometry/features/base.py @@ -0,0 +1,99 @@ +"""Вспомогательные функции для вычисления признаков.""" + +from __future__ import annotations + +import math +import re +from collections import Counter +from typing import Iterable, Sequence + +import numpy as np +import pandas as pd + + +def safe_div(numerator: float, denominator: float, default: float = 0.0) -> float: + """Деление без исключения при нулевом знаменателе.""" + if denominator == 0: + return default + return float(numerator) / float(denominator) + + +def finite(value: float, default: float = 0.0) -> float: + """Заменяет NaN и бесконечности на допустимое число.""" + try: + result = float(value) + except (TypeError, ValueError): + return default + return result if math.isfinite(result) else default + + +def as_list(row: pd.Series, column: str) -> list: + value = row.get(column, []) + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + if value is None or (isinstance(value, float) and math.isnan(value)): + return [] + return list(value) + + +def word_tokens(tokens: Sequence[str]) -> list[str]: + """Токены, содержащие хотя бы одну букву или цифру.""" + return [token for token in tokens if re.search(r"[A-Za-z0-9]", token)] + + +def alphabetic_tokens(tokens: Sequence[str]) -> list[str]: + return [token for token in tokens if re.search(r"[A-Za-z]", token)] + + +def lower_tokens(tokens: Sequence[str]) -> list[str]: + return [token.lower() for token in tokens] + + +def sentence_word_counts(row: pd.Series) -> list[int]: + sentences = as_list(row, "sentences") + counts = [len(word_tokens(sentence)) for sentence in sentences] + counts = [count for count in counts if count > 0] + return counts or [len(word_tokens(as_list(row, "tokens")))] + + +def count_set(tokens: Iterable[str], vocabulary: set[str]) -> int: + return sum(1 for token in tokens if token.lower() in vocabulary) + + +def rate_set(tokens: Sequence[str], vocabulary: set[str]) -> float: + return safe_div(count_set(tokens, vocabulary), len(tokens)) + + +def counter(tokens: Iterable[str]) -> Counter: + return Counter(token.lower() for token in tokens) + + +def mean(values: Sequence[float]) -> float: + return float(np.mean(values)) if values else 0.0 + + +def std(values: Sequence[float]) -> float: + return float(np.std(values, ddof=0)) if values else 0.0 + + +def median(values: Sequence[float]) -> float: + return float(np.median(values)) if values else 0.0 + + +def percentile(values: Sequence[float], q: float) -> float: + return float(np.percentile(values, q)) if values else 0.0 + + +def word_lengths(tokens: Sequence[str]) -> list[int]: + lengths: list[int] = [] + for token in tokens: + letters = re.sub(r"[^A-Za-z]", "", token) + if letters: + lengths.append(len(letters)) + return lengths + + +def strip_punctuation(token: str) -> str: + return re.sub(r"^[^A-Za-z0-9]+|[^A-Za-z0-9]+$", "", token) diff --git a/lab1/stylometry/features/characters.py b/lab1/stylometry/features/characters.py new file mode 100644 index 0000000..55e68b5 --- /dev/null +++ b/lab1/stylometry/features/characters.py @@ -0,0 +1,38 @@ +"""Символьные признаки: частоты букв, регистр, цифры, пунктуация.""" + +from __future__ import annotations + +import string + +import pandas as pd + +from .base import safe_div + + +def extract(row: pd.Series) -> dict[str, float]: + text = str(row.get("text", "")) + lower = text.lower() + letters = [char for char in lower if "a" <= char <= "z"] + n_letters = max(len(letters), 1) + n_chars = max(len(text), 1) + + features: dict[str, float] = {} + for char in string.ascii_lowercase: + features[f"char_freq_{char}"] = safe_div(letters.count(char), n_letters) + + upper = sum(1 for char in text if char.isupper()) + digits = sum(1 for char in text if char.isdigit()) + spaces = sum(1 for char in text if char.isspace()) + punctuation = sum(1 for char in text if not char.isalnum() and not char.isspace()) + features.update( + { + "char_letter_count": float(len(letters)), + "char_upper_rate": safe_div(upper, n_letters), + "char_digit_rate": safe_div(digits, n_chars), + "char_space_rate": safe_div(spaces, n_chars), + "char_punctuation_rate": safe_div(punctuation, n_chars), + "char_apostrophe_rate": safe_div(text.count("'"), n_chars), + "char_quote_rate": safe_div(text.count('"'), n_chars), + } + ) + return features diff --git a/lab1/stylometry/features/dialogue.py b/lab1/stylometry/features/dialogue.py new file mode 100644 index 0000000..35fdbc4 --- /dev/null +++ b/lab1/stylometry/features/dialogue.py @@ -0,0 +1,98 @@ +"""Признаки, специфичные для игровых реплик и разговорной речи.""" + +from __future__ import annotations + +import re + +import pandas as pd + +from ..lexicons import ( + CONTRACTIONS, + DISCOURSE_MARKERS, + HEDGES, + INTENSIFIERS, + INTERJECTIONS, + NEGATIONS, + POLITENESS, +) +from .base import as_list, safe_div, word_tokens + +_LAUGHTER_RE = re.compile(r"\b(?:ha){2,}\b|\bhehe\b|\bhaha\b|\blol\b", re.IGNORECASE) +_EMOTION_WORDS = { + "love", "hate", "like", "want", "feel", "think", "know", "happy", + "sad", "angry", "afraid", "scared", "excited", "worried", "glad", + "sorry", "hope", "wish", "miss", "enjoy", "fun", "nice", "good", + "bad", "terrible", "awful", "great", "wonderful", +} + + +def extract(row: pd.Series) -> dict[str, float]: + text = str(row.get("text", "")) + tokens = [str(token).lower() for token in as_list(row, "tokens")] + words = word_tokens(tokens) + n_words = max(len(words), 1) + sentences = as_list(row, "sentences") + n_sentences = max(len(sentences), 1) + alphabetic = [token for token in tokens if re.search(r"[A-Za-z]", token)] + n_alpha = max(len(alphabetic), 1) + ellipsis = len(re.findall(r"\.{3,}", text)) + repeated = len(re.findall(r"[!?]{2,}", text)) + + return { + "dlg_placeholder_rate": safe_div(float(row.get("placeholder_count", 0)), n_words), + "dlg_all_caps_rate": safe_div( + sum(1 for token in alphabetic if len(token) > 1 and token.isupper()), n_alpha + ), + "dlg_ellipsis_rate": safe_div(ellipsis, n_sentences), + "dlg_repeated_punctuation_rate": safe_div(repeated, n_sentences), + "dlg_interjection_rate": safe_div( + sum(1 for token in tokens if token in INTERJECTIONS), n_words + ), + "dlg_laughter_rate": safe_div(len(_LAUGHTER_RE.findall(text)), n_sentences), + "dlg_discourse_marker_rate": safe_div( + sum(1 for token in tokens if token in DISCOURSE_MARKERS), n_words + ), + "dlg_hedge_rate": safe_div( + sum(1 for token in tokens if token in HEDGES), n_words + ), + "dlg_intensifier_rate": safe_div( + sum(1 for token in tokens if token in INTENSIFIERS), n_words + ), + "dlg_politeness_rate": safe_div( + sum(1 for token in tokens if token in POLITENESS), n_words + ), + "dlg_negation_rate": safe_div( + sum(1 for token in tokens if token in NEGATIONS), n_words + ), + "dlg_contraction_rate": safe_div( + sum(1 for token in tokens if token in CONTRACTIONS or "'" in token), n_words + ), + "dlg_question_tag_rate": safe_div( + len( + re.findall( + r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$", + text, + flags=re.IGNORECASE, + ) + ), + n_sentences, + ), + "dlg_filler_rate": safe_div( + sum(1 for token in tokens if token in {"um", "uh", "hmm", "mmm", "eh"}), n_words + ), + "dlg_emotion_word_rate": safe_div( + sum(1 for token in tokens if token in _EMOTION_WORDS), n_words + ), + "dlg_first_person_rate": safe_div( + sum(1 for token in tokens if token in {"i", "me", "my", "mine", "myself"}), + n_words, + ), + "dlg_second_person_rate": safe_div( + sum(1 for token in tokens if token in {"you", "your", "yours", "yourself"}), + n_words, + ), + "dlg_one_word_sentence_rate": safe_div( + sum(1 for sentence in sentences if len(word_tokens(sentence)) == 1), + n_sentences, + ), + } diff --git a/lab1/stylometry/features/function_words.py b/lab1/stylometry/features/function_words.py new file mode 100644 index 0000000..b3f63ac --- /dev/null +++ b/lab1/stylometry/features/function_words.py @@ -0,0 +1,46 @@ +"""Относительные частоты функциональных слов. + +Это ядро классической стилометрии: закрытые классы слов мало зависят от +темы, но хорошо отражают авторский или персонажный синтаксис. +""" + +from __future__ import annotations + +import re + +import pandas as pd + +from ..lexicons import FUNCTION_WORD_CATEGORIES, FUNCTION_WORDS +from .base import as_list, safe_div, word_tokens + + +def _safe_name(word: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", word.lower()).strip("_") + + +def _sorted_words() -> list[str]: + return sorted(FUNCTION_WORDS) + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = [token.lower() for token in word_tokens(as_list(row, "tokens"))] + n_words = max(len(tokens), 1) + features: dict[str, float] = {} + + for word in _sorted_words(): + features[f"fw_word_{_safe_name(word)}"] = safe_div(tokens.count(word), n_words) + + for category, vocabulary in FUNCTION_WORD_CATEGORIES.items(): + count = sum(1 for token in tokens if token in vocabulary) + features[f"fw_cat_{category}"] = safe_div(count, n_words) + + features["fw_total_rate"] = safe_div( + sum(1 for token in tokens if token in FUNCTION_WORDS), n_words + ) + features["fw_unique_types"] = float(len(set(tokens) & FUNCTION_WORDS)) + return features + + +def feature_words() -> list[str]: + """Список слов, для которых создаются отдельные признаки.""" + return _sorted_words() diff --git a/lab1/stylometry/features/length.py b/lab1/stylometry/features/length.py new file mode 100644 index 0000000..0bb6bf1 --- /dev/null +++ b/lab1/stylometry/features/length.py @@ -0,0 +1,48 @@ +"""Признаки длины и структуры реплики.""" + +from __future__ import annotations + +import pandas as pd + +from .base import ( + as_list, + finite, + mean, + median, + safe_div, + sentence_word_counts, + std, + word_lengths, + word_tokens, +) + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = as_list(row, "tokens") + words = word_tokens(tokens) + lengths = word_lengths(words) + sentence_lengths = sentence_word_counts(row) + n_sentences = max(len(sentence_lengths), 1) + text = str(row.get("text", "")) + + return { + "len_tokens": float(len(tokens)), + "len_words": float(len(words)), + "len_chars": float(len(text)), + "len_sentences": float(n_sentences), + "len_mean_sentence_words": finite(mean(sentence_lengths)), + "len_std_sentence_words": finite(std(sentence_lengths)), + "len_min_sentence_words": float(min(sentence_lengths)) if sentence_lengths else 0.0, + "len_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0, + "len_median_sentence_words": finite(median(sentence_lengths)), + "len_pct_sentences_lt5": safe_div(sum(1 for value in sentence_lengths if value < 5), n_sentences), + "len_pct_sentences_gt20": safe_div(sum(1 for value in sentence_lengths if value > 20), n_sentences), + "len_pct_one_word_sentences": safe_div(sum(1 for value in sentence_lengths if value == 1), n_sentences), + "len_mean_word_chars": finite(mean(lengths)), + "len_std_word_chars": finite(std(lengths)), + "len_median_word_chars": finite(median(lengths)), + "len_pct_words_ge7": safe_div(sum(1 for value in lengths if value >= 7), len(lengths)), + "len_pct_words_ge10": safe_div(sum(1 for value in lengths if value >= 10), len(lengths)), + "len_pct_words_ge13": safe_div(sum(1 for value in lengths if value >= 13), len(lengths)), + "len_pct_single_letter_words": safe_div(sum(1 for value in lengths if value == 1), len(lengths)), + } diff --git a/lab1/stylometry/features/lexical.py b/lab1/stylometry/features/lexical.py new file mode 100644 index 0000000..78f9865 --- /dev/null +++ b/lab1/stylometry/features/lexical.py @@ -0,0 +1,48 @@ +"""Лексические признаки: длины слов и орфографические особенности.""" + +from __future__ import annotations + +import re + +import pandas as pd + +from .base import as_list, safe_div, word_lengths, word_tokens + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = as_list(row, "tokens") + words = word_tokens(tokens) + lengths = word_lengths(words) + total = max(len(lengths), 1) + + features: dict[str, float] = {} + for size in range(1, 10): + features[f"lex_word_length_{size}"] = safe_div( + sum(1 for value in lengths if value == size), total + ) + features["lex_word_length_10plus"] = safe_div( + sum(1 for value in lengths if value >= 10), total + ) + + alphabetic = [token for token in tokens if re.search(r"[A-Za-z]", token)] + n_alpha = max(len(alphabetic), 1) + features.update( + { + "lex_pct_apostrophe_words": safe_div( + sum(1 for token in alphabetic if "'" in token), n_alpha + ), + "lex_pct_capitalized_words": safe_div( + sum(1 for token in alphabetic if token[:1].isupper()), n_alpha + ), + "lex_pct_all_caps_words": safe_div( + sum(1 for token in alphabetic if len(token) > 1 and token.isupper()), n_alpha + ), + "lex_pct_hyphenated_words": safe_div( + sum(1 for token in alphabetic if "-" in token), n_alpha + ), + "lex_pct_digit_words": safe_div( + sum(1 for token in alphabetic if any(char.isdigit() for char in token)), n_alpha + ), + } + ) + return features diff --git a/lab1/stylometry/features/pos_features.py b/lab1/stylometry/features/pos_features.py new file mode 100644 index 0000000..aec09b5 --- /dev/null +++ b/lab1/stylometry/features/pos_features.py @@ -0,0 +1,61 @@ +"""Частеречные признаки на основе NLTK. + +Используются две системы тегов: Universal POS (крупные классы) и Penn +Treebank (более дробные). Дополнительно считаются отношения и отдельные +грамматические категории. +""" + +from __future__ import annotations + +import pandas as pd + +from ..lexicons import MODALS +from ..tokenization import ptb_tag_list, upos_tag_list +from .base import as_list, safe_div, word_tokens + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = as_list(row, "tokens") + upos = as_list(row, "upos") + ptb = as_list(row, "ptb") + lower = [str(token).lower() for token in tokens] + n_words = max(len(word_tokens(tokens)), 1) + + features: dict[str, float] = {} + for tag in upos_tag_list(): + features[f"pos_upos_{tag}"] = safe_div(sum(1 for value in upos if value == tag), n_words) + for tag in ptb_tag_list(): + features[f"pos_ptb_{tag}"] = safe_div(sum(1 for value in ptb if value == tag), n_words) + + noun = sum(1 for value in upos if value == "NOUN") + verb = sum(1 for value in upos if value == "VERB") + adjective = sum(1 for value in upos if value == "ADJ") + adverb = sum(1 for value in upos if value == "ADV") + pronoun = sum(1 for value in upos if value == "PRON") + + features.update( + { + "pos_noun_verb_ratio": safe_div(noun + 1.0, verb + 1.0), + "pos_adj_adv_ratio": safe_div(adjective + 1.0, adverb + 1.0), + "pos_modal_rate": safe_div(sum(1 for token in lower if token in MODALS), n_words), + "pos_aux_rate": safe_div(sum(1 for value in upos if value == "AUX"), n_words), + "pos_past_tense_rate": safe_div(sum(1 for value in ptb if value == "VBD"), n_words), + "pos_present_tense_rate": safe_div( + sum(1 for value in ptb if value in {"VBP", "VBZ"}), n_words + ), + "pos_gerund_rate": safe_div(sum(1 for value in ptb if value == "VBG"), n_words), + "pos_participle_rate": safe_div(sum(1 for value in ptb if value == "VBN"), n_words), + "pos_comparative_rate": safe_div( + sum(1 for value in ptb if value in {"JJR", "RBR"}), n_words + ), + "pos_superlative_rate": safe_div( + sum(1 for value in ptb if value in {"JJS", "RBS"}), n_words + ), + "pos_pronoun_rate": safe_div(pronoun, n_words), + "pos_content_word_rate": safe_div( + sum(1 for value in upos if value in {"NOUN", "VERB", "ADJ", "ADV", "NUM"}), + n_words, + ), + } + ) + return features diff --git a/lab1/stylometry/features/pronouns.py b/lab1/stylometry/features/pronouns.py new file mode 100644 index 0000000..248c535 --- /dev/null +++ b/lab1/stylometry/features/pronouns.py @@ -0,0 +1,45 @@ +"""Признаки местоимений и референции к говорящему/собеседнику.""" + +from __future__ import annotations + +import math + +import pandas as pd + +from ..lexicons import PRONOUN_CATEGORIES, WH_WORDS +from .base import as_list, safe_div, word_tokens + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = [token.lower() for token in word_tokens(as_list(row, "tokens"))] + n_words = max(len(tokens), 1) + features: dict[str, float] = {} + + for category, vocabulary in PRONOUN_CATEGORIES.items(): + features[f"pron_{category}_rate"] = safe_div( + sum(1 for token in tokens if token in vocabulary), n_words + ) + + first_singular = sum( + 1 for token in tokens if token in PRONOUN_CATEGORIES["first_singular"] + ) + second = sum(1 for token in tokens if token in PRONOUN_CATEGORIES["second"]) + features["pron_i_you_log_ratio"] = math.log((first_singular + 1.0) / (second + 1.0)) + features["pron_first_person_rate"] = safe_div( + first_singular + + sum(1 for token in tokens if token in PRONOUN_CATEGORIES["first_plural"]), + n_words, + ) + features["pron_third_person_rate"] = safe_div( + sum( + 1 + for token in tokens + if token in PRONOUN_CATEGORIES["third_singular"] + or token in PRONOUN_CATEGORIES["third_plural"] + ), + n_words, + ) + features["pron_wh_rate"] = safe_div( + sum(1 for token in tokens if token in WH_WORDS), n_words + ) + return features diff --git a/lab1/stylometry/features/punctuation.py b/lab1/stylometry/features/punctuation.py new file mode 100644 index 0000000..897e8aa --- /dev/null +++ b/lab1/stylometry/features/punctuation.py @@ -0,0 +1,57 @@ +"""Признаки пунктуации и оформления высказывания.""" + +from __future__ import annotations + +import re + +import pandas as pd + +from .base import as_list, safe_div, word_tokens + + +def extract(row: pd.Series) -> dict[str, float]: + text = str(row.get("text", "")) + sentences = as_list(row, "sentences") + n_sentences = max(len(sentences), 1) + n_words = max(len(word_tokens(as_list(row, "tokens"))), 1) + + ellipsis_count = len(re.findall(r"\.{3,}", text)) + single_periods = len(re.findall(r"(? int: + """Эвристическая оценка числа слогов в английском слове.""" + word = re.sub(r"[^a-z]", "", word.lower()) + if not word: + return 0 + if len(word) <= 3: + return 1 + if word.endswith("e") and not word.endswith("le"): + word = word[:-1] + groups = re.findall(r"[aeiouy]+", word) + return max(len(groups), 1) + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = word_tokens(as_list(row, "tokens")) + words = [re.sub(r"[^A-Za-z]", "", token).lower() for token in tokens] + words = [word for word in words if word] + n_words = max(len(words), 1) + sentence_lengths = sentence_word_counts(row) + n_sentences = max(len(sentence_lengths), 1) + syllables = [count_syllables(word) for word in words] + n_syllables = sum(syllables) + polysyllables = sum(1 for value in syllables if value >= 3) + n_letters = sum(len(word) for word in words) + + words_per_sentence = safe_div(n_words, n_sentences) + syllables_per_word = safe_div(n_syllables, n_words) + polysyllable_rate = safe_div(polysyllables, n_words) + letters_per_word = safe_div(n_letters, n_words) + + flesch = 206.835 - 1.015 * words_per_sentence - 84.6 * syllables_per_word + fk_grade = 0.39 * words_per_sentence + 11.8 * syllables_per_word - 15.59 + gunning_fog = 0.4 * (words_per_sentence + 100.0 * polysyllable_rate) + smog = 1.0430 * (30.0 * safe_div(polysyllables, n_sentences)) ** 0.5 + 3.1291 + coleman_liau = 0.0588 * (letters_per_word * 100.0) - 0.296 * ( + safe_div(n_sentences, n_words) * 100.0 + ) - 15.8 + ari = 4.71 * letters_per_word + 0.5 * words_per_sentence - 21.43 + + return { + "read_syllables_per_word": finite(syllables_per_word), + "read_polysyllable_rate": finite(polysyllable_rate), + "read_flesch": finite(flesch), + "read_fk_grade": finite(fk_grade), + "read_gunning_fog": finite(gunning_fog), + "read_smog": finite(smog), + "read_coleman_liau": finite(coleman_liau), + "read_ari": finite(ari), + "read_words_per_sentence": finite(words_per_sentence), + } diff --git a/lab1/stylometry/features/stylometry.py b/lab1/stylometry/features/stylometry.py new file mode 100644 index 0000000..5c2a78e --- /dev/null +++ b/lab1/stylometry/features/stylometry.py @@ -0,0 +1,192 @@ +"""Агрегатор ручных стилометрических признаков. + +Класс не обучается на данных: все формулы фиксированы. Поэтому его можно +безопасно применять ко всей таблице до кросс-валидации. Обучаемые +преобразования (TF-IDF, SVD, масштабирование) вынесены в другие классы и +настраиваются только на обучающих блоках. +""" + +from __future__ import annotations + +from typing import Callable, Iterable, Sequence + +import numpy as np +import pandas as pd +from sklearn.base import BaseEstimator, TransformerMixin + +from . import ( + characters, + dialogue, + function_words, + length, + lexical, + pos_features, + pronouns, + punctuation, + readability, + syntax, + vocabulary, +) + +FEATURE_GROUPS: dict[str, Callable[[pd.Series], dict[str, float]]] = { + "length": length.extract, + "lexical": lexical.extract, + "vocabulary": vocabulary.extract, + "characters": characters.extract, + "punctuation": punctuation.extract, + "function_words": function_words.extract, + "pronouns": pronouns.extract, + "pos": pos_features.extract, + "syntax": syntax.extract, + "readability": readability.extract, + "dialogue": dialogue.extract, +} + +FEATURE_GROUP_PREFIXES: dict[str, str] = { + "length": "len_", + "lexical": "lex_", + "vocabulary": "vocab_", + "characters": "char_", + "punctuation": "punct_", + "function_words": "fw_", + "pronouns": "pron_", + "pos": "pos_", + "syntax": "syn_", + "readability": "read_", + "dialogue": "dlg_", +} + +EXPLORATORY_GROUPS = {"vocabulary", "readability"} + + +def default_groups() -> list[str]: + return list(FEATURE_GROUPS) + + +def _validate_groups(groups: Sequence[str]) -> list[str]: + unknown = [group for group in groups if group not in FEATURE_GROUPS] + if unknown: + raise ValueError( + f"Неизвестные группы признаков: {unknown}. " + f"Доступны: {sorted(FEATURE_GROUPS)}" + ) + return list(groups) + + +class StylometricVectorizer(BaseEstimator, TransformerMixin): + """Считает матрицу ручных признаков по таблице реплик. + + Ожидает DataFrame с колонками ``tokens``, ``upos``, ``ptb``, ``sentences``, + ``text`` и ``placeholder_count``. Такие колонки создаёт + :class:`stylometry.tokenization.Tokenizer`. + """ + + def __init__(self, groups: Sequence[str] | None = None) -> None: + self.groups = groups + + def _selected_groups(self) -> list[str]: + groups = self.groups if self.groups is not None else default_groups() + return _validate_groups(groups) + + @staticmethod + def _extract_row(row: pd.Series, groups: Sequence[str]) -> dict[str, float]: + features: dict[str, float] = {} + for group in groups: + features.update(FEATURE_GROUPS[group](row)) + return features + + def fit(self, X: pd.DataFrame, y=None) -> "StylometricVectorizer": + if not isinstance(X, pd.DataFrame): + raise TypeError("StylometricVectorizer ожидает pandas.DataFrame.") + if len(X) == 0: + raise ValueError("Пустая таблица признаков.") + groups = self._selected_groups() + sample = self._extract_row(X.iloc[0], groups) + self.feature_names_in_ = np.array(list(sample.keys()), dtype=object) + self.n_features_in_ = len(self.feature_names_in_) + return self + + def transform(self, X: pd.DataFrame) -> np.ndarray: + if not isinstance(X, pd.DataFrame): + raise TypeError("StylometricVectorizer ожидает pandas.DataFrame.") + groups = self._selected_groups() + rows: list[list[float]] = [] + names: list[str] | None = None + for _, row in X.iterrows(): + features = self._extract_row(row, groups) + if names is None: + names = list(features.keys()) + rows.append([float(features.get(name, 0.0)) for name in names]) + if names is None: + raise ValueError("Не удалось вычислить признаки: таблица пуста.") + self.feature_names_in_ = np.array(names, dtype=object) + self.n_features_in_ = len(names) + return np.asarray(rows, dtype=float) + + def get_feature_names_out(self, input_features=None) -> np.ndarray: + if not hasattr(self, "feature_names_in_"): + raise RuntimeError("Сначала вызовите fit или transform.") + return self.feature_names_in_ + + +def feature_group(feature_name: str) -> str: + """Определяет группу признака по его префиксу.""" + for group, prefix in FEATURE_GROUP_PREFIXES.items(): + if feature_name.startswith(prefix): + return group + return "other" + + +def describe_feature(feature_name: str) -> str: + """Возвращает краткое описание признака для словаря и отчёта.""" + group = feature_group(feature_name) + if group == "function_words": + if feature_name.startswith("fw_word_"): + word = feature_name.removeprefix("fw_word_").replace("_", " ") + return f"Относительная частота функционального слова «{word}»." + if feature_name.startswith("fw_cat_"): + category = feature_name.removeprefix("fw_cat_") + return f"Относительная частота категории функциональных слов «{category}»." + return "Суммарная относительная частота функциональных слов." + if group == "characters" and feature_name.startswith("char_freq_"): + letter = feature_name.removeprefix("char_freq_") + return f"Относительная частота буквы «{letter}» среди латинских букв." + if group == "pos" and feature_name.startswith("pos_upos_"): + tag = feature_name.removeprefix("pos_upos_") + return f"Доля универсального частеречного тега {tag}." + if group == "pos" and feature_name.startswith("pos_ptb_"): + tag = feature_name.removeprefix("pos_ptb_") + return f"Доля тега Penn Treebank {tag}." + if group == "lexical" and feature_name.startswith("lex_word_length_"): + size = feature_name.removeprefix("lex_word_length_") + return f"Доля слов длины {size} (в символах)." + descriptions = { + "length": "Признак длины или структуры реплики.", + "lexical": "Лексический признак длины слова или орфографии.", + "vocabulary": "Признак словарного богатства; нестабилен на коротких текстах.", + "characters": "Символьный признак: регистр, цифры, пунктуация, буквы.", + "punctuation": "Признак пунктуационного оформления.", + "function_words": "Признак функциональных слов.", + "pronouns": "Признак местоимений и референции.", + "pos": "Частеречный признак.", + "syntax": "Синтаксический признак-прокси.", + "readability": "Метрика читаемости; на коротких репликах приблизительна.", + "dialogue": "Признак разговорной речи и игровых реплик.", + } + return descriptions.get(group, "Стилометрический признак.") + + +def feature_dictionary(feature_names: Iterable[str]) -> pd.DataFrame: + """Строит таблицу с описанием признаков для отчёта.""" + rows = [] + for name in feature_names: + group = feature_group(name) + rows.append( + { + "feature": name, + "group": group, + "description": describe_feature(name), + "status": "exploratory" if group in EXPLORATORY_GROUPS else "core", + } + ) + return pd.DataFrame(rows) diff --git a/lab1/stylometry/features/syntax.py b/lab1/stylometry/features/syntax.py new file mode 100644 index 0000000..33ac80f --- /dev/null +++ b/lab1/stylometry/features/syntax.py @@ -0,0 +1,112 @@ +"""Синтаксические признаки-прокси. + +Полноценный синтаксический анализ не используется: для коротких реплик он +нестабилен и требует тяжёлых моделей. Вместо этого считаются устойчивые +приближения по функциям слов, длине предложений и окончаниям. +""" + +from __future__ import annotations + +import re + +import pandas as pd + +from ..lexicons import COORDINATORS, MODALS, NEGATIONS, SUBORDINATORS +from .base import as_list, safe_div, sentence_word_counts, word_tokens + +_IMPERATIVE_VERBS = { + "go", "come", "look", "take", "get", "give", "tell", "let", "stop", + "wait", "listen", "try", "keep", "put", "make", "do", "be", "have", + "see", "say", "help", "leave", "stay", "bring", "show", "find", +} + + +def _sentence_first_words(row: pd.Series) -> list[str]: + sentences = as_list(row, "sentences") + first_words: list[str] = [] + for sentence in sentences: + for token in sentence: + if re.search(r"[A-Za-z]", token): + first_words.append(token.lower()) + break + return first_words + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = [str(token).lower() for token in as_list(row, "tokens")] + words = word_tokens(tokens) + n_words = max(len(words), 1) + sentences = as_list(row, "sentences") + n_sentences = max(len(sentences), 1) + sentence_lengths = sentence_word_counts(row) + first_words = _sentence_first_words(row) + text = str(row.get("text", "")) + + coordinators = sum(1 for token in tokens if token in COORDINATORS) + subordinators = sum(1 for token in tokens if token in SUBORDINATORS) + clauses = coordinators + subordinators + + passive = len( + re.findall( + r"\b(?:be|is|are|was|were|been|being)\s+" + r"(?:\w+ed|known|made|given|taken|seen|done|written|spoken)\b", + text, + flags=re.IGNORECASE, + ) + ) + question_tags = len( + re.findall( + r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$", + text, + flags=re.IGNORECASE, + ) + ) + imperative = sum(1 for word in first_words if word in _IMPERATIVE_VERBS) + sentence_initial_conjunction = sum( + 1 for word in first_words if word in COORDINATORS or word in SUBORDINATORS + ) + + return { + "syn_mean_clauses_per_sentence": safe_div(clauses, n_sentences), + "syn_coordinator_per_sentence": safe_div(coordinators, n_sentences), + "syn_subordinator_per_sentence": safe_div(subordinators, n_sentences), + "syn_sentence_initial_conjunction_rate": safe_div( + sentence_initial_conjunction, n_sentences + ), + "syn_question_rate": safe_div( + sum( + 1 + for sentence in sentences + if sentence and " ".join(sentence).rstrip().endswith("?") + ), + n_sentences, + ), + "syn_exclamation_rate": safe_div( + sum( + 1 + for sentence in sentences + if sentence and " ".join(sentence).rstrip().endswith("!") + ), + n_sentences, + ), + "syn_modal_rate": safe_div(sum(1 for token in tokens if token in MODALS), n_words), + "syn_negation_per_sentence": safe_div( + sum(1 for token in tokens if token in NEGATIONS), n_sentences + ), + "syn_that_rate": safe_div(tokens.count("that"), n_words), + "syn_to_rate": safe_div(tokens.count("to"), n_words), + "syn_ing_rate": safe_div( + sum(1 for token in words if token.endswith("ing")), n_words + ), + "syn_ed_rate": safe_div( + sum(1 for token in words if token.endswith("ed")), n_words + ), + "syn_ly_rate": safe_div( + sum(1 for token in words if token.endswith("ly")), n_words + ), + "syn_passive_proxy_per_sentence": safe_div(passive, n_sentences), + "syn_question_tag_per_sentence": safe_div(question_tags, n_sentences), + "syn_imperative_proxy_rate": safe_div(imperative, n_sentences), + "syn_mean_sentence_words": safe_div(sum(sentence_lengths), n_sentences), + "syn_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0, + } diff --git a/lab1/stylometry/features/vocabulary.py b/lab1/stylometry/features/vocabulary.py new file mode 100644 index 0000000..4714bff --- /dev/null +++ b/lab1/stylometry/features/vocabulary.py @@ -0,0 +1,67 @@ +"""Признаки словарного богатства. + +На коротких репликах эти признаки нестабильны. Они включены как +дополнительная группа и должны проверяться отдельно в ablation-эксперименте. +""" + +from __future__ import annotations + +import math +from collections import Counter + +import pandas as pd + +from .base import as_list, finite, safe_div, word_tokens + + +def _yule_k(counter: Counter[str], n_tokens: int) -> float: + if n_tokens <= 1: + return 0.0 + frequency_of_frequencies = Counter(counter.values()) + second_moment = sum( + frequency * frequency * count + for frequency, count in frequency_of_frequencies.items() + ) + return finite(10_000.0 * (second_moment - n_tokens) / (n_tokens * n_tokens)) + + +def _simpson_d(counter: Counter[str], n_tokens: int) -> float: + if n_tokens <= 1: + return 0.0 + numerator = sum(count * (count - 1) for count in counter.values()) + return finite(numerator / (n_tokens * (n_tokens - 1))) + + +def extract(row: pd.Series) -> dict[str, float]: + tokens = [token.lower() for token in word_tokens(as_list(row, "tokens"))] + counter = Counter(tokens) + n_tokens = len(tokens) + n_types = len(counter) + hapax = sum(1 for count in counter.values() if count == 1) + dis = sum(1 for count in counter.values() if count == 2) + + ttr = safe_div(n_types, n_tokens) + guiraud = safe_div(n_types, math.sqrt(n_tokens)) + herdan = finite(math.log(n_types) / math.log(n_tokens)) if n_tokens > 1 and n_types > 1 else 0.0 + hapax_ratio = safe_div(hapax, n_types) + dis_ratio = safe_div(dis, n_types) + if n_types > 0 and hapax_ratio < 1.0: + honore = finite(100.0 * math.log(n_tokens) / (1.0 - hapax_ratio)) + else: + honore = 0.0 + + top10 = sum(count for _, count in counter.most_common(10)) + return { + "vocab_tokens": float(n_tokens), + "vocab_types": float(n_types), + "vocab_ttr": ttr, + "vocab_guiraud": finite(guiraud), + "vocab_herdan": herdan, + "vocab_yule_k": _yule_k(counter, n_tokens), + "vocab_simpson_d": _simpson_d(counter, n_tokens), + "vocab_hapax_ratio": hapax_ratio, + "vocab_dis_ratio": dis_ratio, + "vocab_honore_r": honore, + "vocab_top10_coverage": safe_div(top10, n_tokens), + "vocab_mean_word_frequency": safe_div(n_tokens, n_types), + }