113 lines
4.3 KiB
Python
113 lines
4.3 KiB
Python
"""Синтаксические признаки-прокси.
|
|
|
|
Полноценный синтаксический анализ не используется: для коротких реплик он
|
|
нестабилен и требует тяжёлых моделей. Вместо этого считаются устойчивые
|
|
приближения по функциям слов, длине предложений и окончаниям.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pandas as pd
|
|
|
|
from ..lexicons import COORDINATORS, MODALS, NEGATIONS, SUBORDINATORS
|
|
from .base import as_list, safe_div, sentence_word_counts, word_tokens
|
|
|
|
_IMPERATIVE_VERBS = {
|
|
"go", "come", "look", "take", "get", "give", "tell", "let", "stop",
|
|
"wait", "listen", "try", "keep", "put", "make", "do", "be", "have",
|
|
"see", "say", "help", "leave", "stay", "bring", "show", "find",
|
|
}
|
|
|
|
|
|
def _sentence_first_words(row: pd.Series) -> list[str]:
|
|
sentences = as_list(row, "sentences")
|
|
first_words: list[str] = []
|
|
for sentence in sentences:
|
|
for token in sentence:
|
|
if re.search(r"[A-Za-z]", token):
|
|
first_words.append(token.lower())
|
|
break
|
|
return first_words
|
|
|
|
|
|
def extract(row: pd.Series) -> dict[str, float]:
|
|
tokens = [str(token).lower() for token in as_list(row, "tokens")]
|
|
words = word_tokens(tokens)
|
|
n_words = max(len(words), 1)
|
|
sentences = as_list(row, "sentences")
|
|
n_sentences = max(len(sentences), 1)
|
|
sentence_lengths = sentence_word_counts(row)
|
|
first_words = _sentence_first_words(row)
|
|
text = str(row.get("text", ""))
|
|
|
|
coordinators = sum(1 for token in tokens if token in COORDINATORS)
|
|
subordinators = sum(1 for token in tokens if token in SUBORDINATORS)
|
|
clauses = coordinators + subordinators
|
|
|
|
passive = len(
|
|
re.findall(
|
|
r"\b(?:be|is|are|was|were|been|being)\s+"
|
|
r"(?:\w+ed|known|made|given|taken|seen|done|written|spoken)\b",
|
|
text,
|
|
flags=re.IGNORECASE,
|
|
)
|
|
)
|
|
question_tags = len(
|
|
re.findall(
|
|
r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$",
|
|
text,
|
|
flags=re.IGNORECASE,
|
|
)
|
|
)
|
|
imperative = sum(1 for word in first_words if word in _IMPERATIVE_VERBS)
|
|
sentence_initial_conjunction = sum(
|
|
1 for word in first_words if word in COORDINATORS or word in SUBORDINATORS
|
|
)
|
|
|
|
return {
|
|
"syn_mean_clauses_per_sentence": safe_div(clauses, n_sentences),
|
|
"syn_coordinator_per_sentence": safe_div(coordinators, n_sentences),
|
|
"syn_subordinator_per_sentence": safe_div(subordinators, n_sentences),
|
|
"syn_sentence_initial_conjunction_rate": safe_div(
|
|
sentence_initial_conjunction, n_sentences
|
|
),
|
|
"syn_question_rate": safe_div(
|
|
sum(
|
|
1
|
|
for sentence in sentences
|
|
if sentence and " ".join(sentence).rstrip().endswith("?")
|
|
),
|
|
n_sentences,
|
|
),
|
|
"syn_exclamation_rate": safe_div(
|
|
sum(
|
|
1
|
|
for sentence in sentences
|
|
if sentence and " ".join(sentence).rstrip().endswith("!")
|
|
),
|
|
n_sentences,
|
|
),
|
|
"syn_modal_rate": safe_div(sum(1 for token in tokens if token in MODALS), n_words),
|
|
"syn_negation_per_sentence": safe_div(
|
|
sum(1 for token in tokens if token in NEGATIONS), n_sentences
|
|
),
|
|
"syn_that_rate": safe_div(tokens.count("that"), n_words),
|
|
"syn_to_rate": safe_div(tokens.count("to"), n_words),
|
|
"syn_ing_rate": safe_div(
|
|
sum(1 for token in words if token.endswith("ing")), n_words
|
|
),
|
|
"syn_ed_rate": safe_div(
|
|
sum(1 for token in words if token.endswith("ed")), n_words
|
|
),
|
|
"syn_ly_rate": safe_div(
|
|
sum(1 for token in words if token.endswith("ly")), n_words
|
|
),
|
|
"syn_passive_proxy_per_sentence": safe_div(passive, n_sentences),
|
|
"syn_question_tag_per_sentence": safe_div(question_tags, n_sentences),
|
|
"syn_imperative_proxy_rate": safe_div(imperative, n_sentences),
|
|
"syn_mean_sentence_words": safe_div(sum(sentence_lengths), n_sentences),
|
|
"syn_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0,
|
|
}
|