"""Синтаксические признаки-прокси. Полноценный синтаксический анализ не используется: для коротких реплик он нестабилен и требует тяжёлых моделей. Вместо этого считаются устойчивые приближения по функциям слов, длине предложений и окончаниям. """ from __future__ import annotations import re import pandas as pd from ..lexicons import COORDINATORS, MODALS, NEGATIONS, SUBORDINATORS from .base import as_list, safe_div, sentence_word_counts, word_tokens _IMPERATIVE_VERBS = { "go", "come", "look", "take", "get", "give", "tell", "let", "stop", "wait", "listen", "try", "keep", "put", "make", "do", "be", "have", "see", "say", "help", "leave", "stay", "bring", "show", "find", } def _sentence_first_words(row: pd.Series) -> list[str]: sentences = as_list(row, "sentences") first_words: list[str] = [] for sentence in sentences: for token in sentence: if re.search(r"[A-Za-z]", token): first_words.append(token.lower()) break return first_words def extract(row: pd.Series) -> dict[str, float]: tokens = [str(token).lower() for token in as_list(row, "tokens")] words = word_tokens(tokens) n_words = max(len(words), 1) sentences = as_list(row, "sentences") n_sentences = max(len(sentences), 1) sentence_lengths = sentence_word_counts(row) first_words = _sentence_first_words(row) text = str(row.get("text", "")) coordinators = sum(1 for token in tokens if token in COORDINATORS) subordinators = sum(1 for token in tokens if token in SUBORDINATORS) clauses = coordinators + subordinators passive = len( re.findall( r"\b(?:be|is|are|was|were|been|being)\s+" r"(?:\w+ed|known|made|given|taken|seen|done|written|spoken)\b", text, flags=re.IGNORECASE, ) ) question_tags = len( re.findall( r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$", text, flags=re.IGNORECASE, ) ) imperative = sum(1 for word in first_words if word in _IMPERATIVE_VERBS) sentence_initial_conjunction = sum( 1 for word in first_words if word in COORDINATORS or word in SUBORDINATORS ) return { "syn_mean_clauses_per_sentence": safe_div(clauses, n_sentences), "syn_coordinator_per_sentence": safe_div(coordinators, n_sentences), "syn_subordinator_per_sentence": safe_div(subordinators, n_sentences), "syn_sentence_initial_conjunction_rate": safe_div( sentence_initial_conjunction, n_sentences ), "syn_question_rate": safe_div( sum( 1 for sentence in sentences if sentence and " ".join(sentence).rstrip().endswith("?") ), n_sentences, ), "syn_exclamation_rate": safe_div( sum( 1 for sentence in sentences if sentence and " ".join(sentence).rstrip().endswith("!") ), n_sentences, ), "syn_modal_rate": safe_div(sum(1 for token in tokens if token in MODALS), n_words), "syn_negation_per_sentence": safe_div( sum(1 for token in tokens if token in NEGATIONS), n_sentences ), "syn_that_rate": safe_div(tokens.count("that"), n_words), "syn_to_rate": safe_div(tokens.count("to"), n_words), "syn_ing_rate": safe_div( sum(1 for token in words if token.endswith("ing")), n_words ), "syn_ed_rate": safe_div( sum(1 for token in words if token.endswith("ed")), n_words ), "syn_ly_rate": safe_div( sum(1 for token in words if token.endswith("ly")), n_words ), "syn_passive_proxy_per_sentence": safe_div(passive, n_sentences), "syn_question_tag_per_sentence": safe_div(question_tags, n_sentences), "syn_imperative_proxy_rate": safe_div(imperative, n_sentences), "syn_mean_sentence_words": safe_div(sum(sentence_lengths), n_sentences), "syn_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0, }