"""Лексические признаки: длины слов и орфографические особенности.""" from __future__ import annotations import re import pandas as pd from .base import as_list, safe_div, word_lengths, word_tokens def extract(row: pd.Series) -> dict[str, float]: tokens = as_list(row, "tokens") words = word_tokens(tokens) lengths = word_lengths(words) total = max(len(lengths), 1) features: dict[str, float] = {} for size in range(1, 10): features[f"lex_word_length_{size}"] = safe_div( sum(1 for value in lengths if value == size), total ) features["lex_word_length_10plus"] = safe_div( sum(1 for value in lengths if value >= 10), total ) alphabetic = [token for token in tokens if re.search(r"[A-Za-z]", token)] n_alpha = max(len(alphabetic), 1) features.update( { "lex_pct_apostrophe_words": safe_div( sum(1 for token in alphabetic if "'" in token), n_alpha ), "lex_pct_capitalized_words": safe_div( sum(1 for token in alphabetic if token[:1].isupper()), n_alpha ), "lex_pct_all_caps_words": safe_div( sum(1 for token in alphabetic if len(token) > 1 and token.isupper()), n_alpha ), "lex_pct_hyphenated_words": safe_div( sum(1 for token in alphabetic if "-" in token), n_alpha ), "lex_pct_digit_words": safe_div( sum(1 for token in alphabetic if any(char.isdigit() for char in token)), n_alpha ), } ) return features