100 lines
3.0 KiB
Python
100 lines
3.0 KiB
Python
"""Вспомогательные функции для вычисления признаков."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import re
|
|
from collections import Counter
|
|
from typing import Iterable, Sequence
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
|
|
def safe_div(numerator: float, denominator: float, default: float = 0.0) -> float:
|
|
"""Деление без исключения при нулевом знаменателе."""
|
|
if denominator == 0:
|
|
return default
|
|
return float(numerator) / float(denominator)
|
|
|
|
|
|
def finite(value: float, default: float = 0.0) -> float:
|
|
"""Заменяет NaN и бесконечности на допустимое число."""
|
|
try:
|
|
result = float(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
return result if math.isfinite(result) else default
|
|
|
|
|
|
def as_list(row: pd.Series, column: str) -> list:
|
|
value = row.get(column, [])
|
|
if isinstance(value, list):
|
|
return value
|
|
if isinstance(value, tuple):
|
|
return list(value)
|
|
if value is None or (isinstance(value, float) and math.isnan(value)):
|
|
return []
|
|
return list(value)
|
|
|
|
|
|
def word_tokens(tokens: Sequence[str]) -> list[str]:
|
|
"""Токены, содержащие хотя бы одну букву или цифру."""
|
|
return [token for token in tokens if re.search(r"[A-Za-z0-9]", token)]
|
|
|
|
|
|
def alphabetic_tokens(tokens: Sequence[str]) -> list[str]:
|
|
return [token for token in tokens if re.search(r"[A-Za-z]", token)]
|
|
|
|
|
|
def lower_tokens(tokens: Sequence[str]) -> list[str]:
|
|
return [token.lower() for token in tokens]
|
|
|
|
|
|
def sentence_word_counts(row: pd.Series) -> list[int]:
|
|
sentences = as_list(row, "sentences")
|
|
counts = [len(word_tokens(sentence)) for sentence in sentences]
|
|
counts = [count for count in counts if count > 0]
|
|
return counts or [len(word_tokens(as_list(row, "tokens")))]
|
|
|
|
|
|
def count_set(tokens: Iterable[str], vocabulary: set[str]) -> int:
|
|
return sum(1 for token in tokens if token.lower() in vocabulary)
|
|
|
|
|
|
def rate_set(tokens: Sequence[str], vocabulary: set[str]) -> float:
|
|
return safe_div(count_set(tokens, vocabulary), len(tokens))
|
|
|
|
|
|
def counter(tokens: Iterable[str]) -> Counter:
|
|
return Counter(token.lower() for token in tokens)
|
|
|
|
|
|
def mean(values: Sequence[float]) -> float:
|
|
return float(np.mean(values)) if values else 0.0
|
|
|
|
|
|
def std(values: Sequence[float]) -> float:
|
|
return float(np.std(values, ddof=0)) if values else 0.0
|
|
|
|
|
|
def median(values: Sequence[float]) -> float:
|
|
return float(np.median(values)) if values else 0.0
|
|
|
|
|
|
def percentile(values: Sequence[float], q: float) -> float:
|
|
return float(np.percentile(values, q)) if values else 0.0
|
|
|
|
|
|
def word_lengths(tokens: Sequence[str]) -> list[int]:
|
|
lengths: list[int] = []
|
|
for token in tokens:
|
|
letters = re.sub(r"[^A-Za-z]", "", token)
|
|
if letters:
|
|
lengths.append(len(letters))
|
|
return lengths
|
|
|
|
|
|
def strip_punctuation(token: str) -> str:
|
|
return re.sub(r"^[^A-Za-z0-9]+|[^A-Za-z0-9]+$", "", token)
|