99 lines
3.6 KiB
Python
99 lines
3.6 KiB
Python
"""Признаки, специфичные для игровых реплик и разговорной речи."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pandas as pd
|
|
|
|
from ..lexicons import (
|
|
CONTRACTIONS,
|
|
DISCOURSE_MARKERS,
|
|
HEDGES,
|
|
INTENSIFIERS,
|
|
INTERJECTIONS,
|
|
NEGATIONS,
|
|
POLITENESS,
|
|
)
|
|
from .base import as_list, safe_div, word_tokens
|
|
|
|
_LAUGHTER_RE = re.compile(r"\b(?:ha){2,}\b|\bhehe\b|\bhaha\b|\blol\b", re.IGNORECASE)
|
|
_EMOTION_WORDS = {
|
|
"love", "hate", "like", "want", "feel", "think", "know", "happy",
|
|
"sad", "angry", "afraid", "scared", "excited", "worried", "glad",
|
|
"sorry", "hope", "wish", "miss", "enjoy", "fun", "nice", "good",
|
|
"bad", "terrible", "awful", "great", "wonderful",
|
|
}
|
|
|
|
|
|
def extract(row: pd.Series) -> dict[str, float]:
|
|
text = str(row.get("text", ""))
|
|
tokens = [str(token).lower() for token in as_list(row, "tokens")]
|
|
words = word_tokens(tokens)
|
|
n_words = max(len(words), 1)
|
|
sentences = as_list(row, "sentences")
|
|
n_sentences = max(len(sentences), 1)
|
|
alphabetic = [token for token in tokens if re.search(r"[A-Za-z]", token)]
|
|
n_alpha = max(len(alphabetic), 1)
|
|
ellipsis = len(re.findall(r"\.{3,}", text))
|
|
repeated = len(re.findall(r"[!?]{2,}", text))
|
|
|
|
return {
|
|
"dlg_placeholder_rate": safe_div(float(row.get("placeholder_count", 0)), n_words),
|
|
"dlg_all_caps_rate": safe_div(
|
|
sum(1 for token in alphabetic if len(token) > 1 and token.isupper()), n_alpha
|
|
),
|
|
"dlg_ellipsis_rate": safe_div(ellipsis, n_sentences),
|
|
"dlg_repeated_punctuation_rate": safe_div(repeated, n_sentences),
|
|
"dlg_interjection_rate": safe_div(
|
|
sum(1 for token in tokens if token in INTERJECTIONS), n_words
|
|
),
|
|
"dlg_laughter_rate": safe_div(len(_LAUGHTER_RE.findall(text)), n_sentences),
|
|
"dlg_discourse_marker_rate": safe_div(
|
|
sum(1 for token in tokens if token in DISCOURSE_MARKERS), n_words
|
|
),
|
|
"dlg_hedge_rate": safe_div(
|
|
sum(1 for token in tokens if token in HEDGES), n_words
|
|
),
|
|
"dlg_intensifier_rate": safe_div(
|
|
sum(1 for token in tokens if token in INTENSIFIERS), n_words
|
|
),
|
|
"dlg_politeness_rate": safe_div(
|
|
sum(1 for token in tokens if token in POLITENESS), n_words
|
|
),
|
|
"dlg_negation_rate": safe_div(
|
|
sum(1 for token in tokens if token in NEGATIONS), n_words
|
|
),
|
|
"dlg_contraction_rate": safe_div(
|
|
sum(1 for token in tokens if token in CONTRACTIONS or "'" in token), n_words
|
|
),
|
|
"dlg_question_tag_rate": safe_div(
|
|
len(
|
|
re.findall(
|
|
r"\b(?:right|isn't it|aren't you|don't you|didn't you|huh)\s*[?.!]?\s*$",
|
|
text,
|
|
flags=re.IGNORECASE,
|
|
)
|
|
),
|
|
n_sentences,
|
|
),
|
|
"dlg_filler_rate": safe_div(
|
|
sum(1 for token in tokens if token in {"um", "uh", "hmm", "mmm", "eh"}), n_words
|
|
),
|
|
"dlg_emotion_word_rate": safe_div(
|
|
sum(1 for token in tokens if token in _EMOTION_WORDS), n_words
|
|
),
|
|
"dlg_first_person_rate": safe_div(
|
|
sum(1 for token in tokens if token in {"i", "me", "my", "mine", "myself"}),
|
|
n_words,
|
|
),
|
|
"dlg_second_person_rate": safe_div(
|
|
sum(1 for token in tokens if token in {"you", "your", "yours", "yourself"}),
|
|
n_words,
|
|
),
|
|
"dlg_one_word_sentence_rate": safe_div(
|
|
sum(1 for sentence in sentences if len(word_tokens(sentence)) == 1),
|
|
n_sentences,
|
|
),
|
|
}
|