49 lines
2.0 KiB
Python
49 lines
2.0 KiB
Python
"""Признаки длины и структуры реплики."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pandas as pd
|
|
|
|
from .base import (
|
|
as_list,
|
|
finite,
|
|
mean,
|
|
median,
|
|
safe_div,
|
|
sentence_word_counts,
|
|
std,
|
|
word_lengths,
|
|
word_tokens,
|
|
)
|
|
|
|
|
|
def extract(row: pd.Series) -> dict[str, float]:
|
|
tokens = as_list(row, "tokens")
|
|
words = word_tokens(tokens)
|
|
lengths = word_lengths(words)
|
|
sentence_lengths = sentence_word_counts(row)
|
|
n_sentences = max(len(sentence_lengths), 1)
|
|
text = str(row.get("text", ""))
|
|
|
|
return {
|
|
"len_tokens": float(len(tokens)),
|
|
"len_words": float(len(words)),
|
|
"len_chars": float(len(text)),
|
|
"len_sentences": float(n_sentences),
|
|
"len_mean_sentence_words": finite(mean(sentence_lengths)),
|
|
"len_std_sentence_words": finite(std(sentence_lengths)),
|
|
"len_min_sentence_words": float(min(sentence_lengths)) if sentence_lengths else 0.0,
|
|
"len_max_sentence_words": float(max(sentence_lengths)) if sentence_lengths else 0.0,
|
|
"len_median_sentence_words": finite(median(sentence_lengths)),
|
|
"len_pct_sentences_lt5": safe_div(sum(1 for value in sentence_lengths if value < 5), n_sentences),
|
|
"len_pct_sentences_gt20": safe_div(sum(1 for value in sentence_lengths if value > 20), n_sentences),
|
|
"len_pct_one_word_sentences": safe_div(sum(1 for value in sentence_lengths if value == 1), n_sentences),
|
|
"len_mean_word_chars": finite(mean(lengths)),
|
|
"len_std_word_chars": finite(std(lengths)),
|
|
"len_median_word_chars": finite(median(lengths)),
|
|
"len_pct_words_ge7": safe_div(sum(1 for value in lengths if value >= 7), len(lengths)),
|
|
"len_pct_words_ge10": safe_div(sum(1 for value in lengths if value >= 10), len(lengths)),
|
|
"len_pct_words_ge13": safe_div(sum(1 for value in lengths if value >= 13), len(lengths)),
|
|
"len_pct_single_letter_words": safe_div(sum(1 for value in lengths if value == 1), len(lengths)),
|
|
}
|