39 lines
1.3 KiB
Python
39 lines
1.3 KiB
Python
"""Символьные признаки: частоты букв, регистр, цифры, пунктуация."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import string
|
|
|
|
import pandas as pd
|
|
|
|
from .base import safe_div
|
|
|
|
|
|
def extract(row: pd.Series) -> dict[str, float]:
|
|
text = str(row.get("text", ""))
|
|
lower = text.lower()
|
|
letters = [char for char in lower if "a" <= char <= "z"]
|
|
n_letters = max(len(letters), 1)
|
|
n_chars = max(len(text), 1)
|
|
|
|
features: dict[str, float] = {}
|
|
for char in string.ascii_lowercase:
|
|
features[f"char_freq_{char}"] = safe_div(letters.count(char), n_letters)
|
|
|
|
upper = sum(1 for char in text if char.isupper())
|
|
digits = sum(1 for char in text if char.isdigit())
|
|
spaces = sum(1 for char in text if char.isspace())
|
|
punctuation = sum(1 for char in text if not char.isalnum() and not char.isspace())
|
|
features.update(
|
|
{
|
|
"char_letter_count": float(len(letters)),
|
|
"char_upper_rate": safe_div(upper, n_letters),
|
|
"char_digit_rate": safe_div(digits, n_chars),
|
|
"char_space_rate": safe_div(spaces, n_chars),
|
|
"char_punctuation_rate": safe_div(punctuation, n_chars),
|
|
"char_apostrophe_rate": safe_div(text.count("'"), n_chars),
|
|
"char_quote_rate": safe_div(text.count('"'), n_chars),
|
|
}
|
|
)
|
|
return features
|