Исправлен .gitignore: пакет stylometry/features добавлен в репозиторий

2026-10-25 15:52:25 +03:00
parent c390224500
commit 10e2444d23
15 changed files with 1000 additions and 7 deletions
--- a/lab1/stylometry/features/syntax.py
+++ b/lab1/stylometry/features/syntax.py
@@ -0,0 +1,112 @@
+"""Синтаксические признаки-прокси.
+
+Полноценный синтаксический анализ не используется: для коротких реплик он
+нестабилен и требует тяжёлых моделей. Вместо этого считаются устойчивые
+приближения по функциям слов, длине предложений и окончаниям.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from ..lexicons import COORDINATORS, MODALS, NEGATIONS, SUBORDINATORS
+from .base import as_list, safe_div, sentence_word_counts, word_tokens
+
+_IMPERATIVE_VERBS = {
+    "go", "come", "look", "take", "get", "give", "tell", "let", "stop",
+    "wait", "listen", "try", "keep", "put", "make", "do", "be", "have",
+    "see", "say", "help", "leave", "stay", "bring", "show", "find",
+}
+
+
+def _sentence_first_words(row: pd.Series) -> list[str]:
+    sentences = as_list(row, "sentences")
+    first_words: list[str] = []
+    for sentence in sentences:
+        for token in sentence:
+            if re.search(r"[A-Za-z]", token):
+                first_words.append(token.lower())
+                break
+    return first_words
+
+
+def extract(row: pd.Series) -> dict[str, float]:
+    tokens = [str(token).lower() for token in as_list(row, "tokens")]
+    words = word_tokens(tokens)
+    n_words = max(len(words), 1)
+    sentences = as_list(row, "sentences")
+    n_sentences = max(len(sentences), 1)
+    sentence_lengths = sentence_word_counts(row)
+    first_words = _sentence_first_words(row)
+    text = str(row.get("text", ""))
+
+    coordinators = sum(1 for token in tokens if token in COORDINATORS)
+    subordinators = sum(1 for token in tokens if token in SUBORDINATORS)
+    clauses = coordinators + subordinators
+
+    passive = len(
+        re.findall(
+            r"\b(?:be|is|are|was|were|been|being)\s+"
+            r"(?:\w+ed|known|made|given|taken|seen|done|written|spoken)\b",
+            text,
+            flags=re.IGNORECASE,
+        )
+    )
+    question_tags = len(
+        re.findall(
+            r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$",
+            text,
+            flags=re.IGNORECASE,
+        )
+    )
+    imperative = sum(1 for word in first_words if word in _IMPERATIVE_VERBS)
+    sentence_initial_conjunction = sum(
+        1 for word in first_words if word in COORDINATORS or word in SUBORDINATORS
+    )
+
+    return {
+        "syn_mean_clauses_per_sentence": safe_div(clauses, n_sentences),
+        "syn_coordinator_per_sentence": safe_div(coordinators, n_sentences),
+        "syn_subordinator_per_sentence": safe_div(subordinators, n_sentences),
+        "syn_sentence_initial_conjunction_rate": safe_div(
+            sentence_initial_conjunction, n_sentences
+        ),
+        "syn_question_rate": safe_div(
+            sum(
+                1
+                for sentence in sentences
+                if sentence and " ".join(sentence).rstrip().endswith("?")
+            ),
+            n_sentences,
+        ),
+        "syn_exclamation_rate": safe_div(
+            sum(
+                1
+                for sentence in sentences
+                if sentence and " ".join(sentence).rstrip().endswith("!")
+            ),
+            n_sentences,
+        ),
+        "syn_modal_rate": safe_div(sum(1 for token in tokens if token in MODALS), n_words),
+        "syn_negation_per_sentence": safe_div(
+            sum(1 for token in tokens if token in NEGATIONS), n_sentences
+        ),
+        "syn_that_rate": safe_div(tokens.count("that"), n_words),
+        "syn_to_rate": safe_div(tokens.count("to"), n_words),
+        "syn_ing_rate": safe_div(
+            sum(1 for token in words if token.endswith("ing")), n_words
+        ),
+        "syn_ed_rate": safe_div(
+            sum(1 for token in words if token.endswith("ed")), n_words
+        ),
+        "syn_ly_rate": safe_div(
+            sum(1 for token in words if token.endswith("ly")), n_words
+        ),
+        "syn_passive_proxy_per_sentence": safe_div(passive, n_sentences),
+        "syn_question_tag_per_sentence": safe_div(question_tags, n_sentences),
+        "syn_imperative_proxy_rate": safe_div(imperative, n_sentences),
+        "syn_mean_sentence_words": safe_div(sum(sentence_lengths), n_sentences),
+        "syn_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0,
+    }