49 lines
1.6 KiB
Python
49 lines
1.6 KiB
Python
"""Лексические признаки: длины слов и орфографические особенности."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pandas as pd
|
|
|
|
from .base import as_list, safe_div, word_lengths, word_tokens
|
|
|
|
|
|
def extract(row: pd.Series) -> dict[str, float]:
|
|
tokens = as_list(row, "tokens")
|
|
words = word_tokens(tokens)
|
|
lengths = word_lengths(words)
|
|
total = max(len(lengths), 1)
|
|
|
|
features: dict[str, float] = {}
|
|
for size in range(1, 10):
|
|
features[f"lex_word_length_{size}"] = safe_div(
|
|
sum(1 for value in lengths if value == size), total
|
|
)
|
|
features["lex_word_length_10plus"] = safe_div(
|
|
sum(1 for value in lengths if value >= 10), total
|
|
)
|
|
|
|
alphabetic = [token for token in tokens if re.search(r"[A-Za-z]", token)]
|
|
n_alpha = max(len(alphabetic), 1)
|
|
features.update(
|
|
{
|
|
"lex_pct_apostrophe_words": safe_div(
|
|
sum(1 for token in alphabetic if "'" in token), n_alpha
|
|
),
|
|
"lex_pct_capitalized_words": safe_div(
|
|
sum(1 for token in alphabetic if token[:1].isupper()), n_alpha
|
|
),
|
|
"lex_pct_all_caps_words": safe_div(
|
|
sum(1 for token in alphabetic if len(token) > 1 and token.isupper()), n_alpha
|
|
),
|
|
"lex_pct_hyphenated_words": safe_div(
|
|
sum(1 for token in alphabetic if "-" in token), n_alpha
|
|
),
|
|
"lex_pct_digit_words": safe_div(
|
|
sum(1 for token in alphabetic if any(char.isdigit() for char in token)), n_alpha
|
|
),
|
|
}
|
|
)
|
|
return features
|