67 lines
2.7 KiB
Python
67 lines
2.7 KiB
Python
"""Метрики читаемости.
|
||
|
||
Формулы адаптированы для коротких разговорных реплик, поэтому значения
|
||
следует рассматривать как относительные признаки, а не как точные оценки
|
||
сложности текста. Число слогов вычисляется эвристически.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
|
||
import pandas as pd
|
||
|
||
from .base import as_list, finite, safe_div, sentence_word_counts, word_tokens
|
||
|
||
|
||
def count_syllables(word: str) -> int:
|
||
"""Эвристическая оценка числа слогов в английском слове."""
|
||
word = re.sub(r"[^a-z]", "", word.lower())
|
||
if not word:
|
||
return 0
|
||
if len(word) <= 3:
|
||
return 1
|
||
if word.endswith("e") and not word.endswith("le"):
|
||
word = word[:-1]
|
||
groups = re.findall(r"[aeiouy]+", word)
|
||
return max(len(groups), 1)
|
||
|
||
|
||
def extract(row: pd.Series) -> dict[str, float]:
|
||
tokens = word_tokens(as_list(row, "tokens"))
|
||
words = [re.sub(r"[^A-Za-z]", "", token).lower() for token in tokens]
|
||
words = [word for word in words if word]
|
||
n_words = max(len(words), 1)
|
||
sentence_lengths = sentence_word_counts(row)
|
||
n_sentences = max(len(sentence_lengths), 1)
|
||
syllables = [count_syllables(word) for word in words]
|
||
n_syllables = sum(syllables)
|
||
polysyllables = sum(1 for value in syllables if value >= 3)
|
||
n_letters = sum(len(word) for word in words)
|
||
|
||
words_per_sentence = safe_div(n_words, n_sentences)
|
||
syllables_per_word = safe_div(n_syllables, n_words)
|
||
polysyllable_rate = safe_div(polysyllables, n_words)
|
||
letters_per_word = safe_div(n_letters, n_words)
|
||
|
||
flesch = 206.835 - 1.015 * words_per_sentence - 84.6 * syllables_per_word
|
||
fk_grade = 0.39 * words_per_sentence + 11.8 * syllables_per_word - 15.59
|
||
gunning_fog = 0.4 * (words_per_sentence + 100.0 * polysyllable_rate)
|
||
smog = 1.0430 * (30.0 * safe_div(polysyllables, n_sentences)) ** 0.5 + 3.1291
|
||
coleman_liau = 0.0588 * (letters_per_word * 100.0) - 0.296 * (
|
||
safe_div(n_sentences, n_words) * 100.0
|
||
) - 15.8
|
||
ari = 4.71 * letters_per_word + 0.5 * words_per_sentence - 21.43
|
||
|
||
return {
|
||
"read_syllables_per_word": finite(syllables_per_word),
|
||
"read_polysyllable_rate": finite(polysyllable_rate),
|
||
"read_flesch": finite(flesch),
|
||
"read_fk_grade": finite(fk_grade),
|
||
"read_gunning_fog": finite(gunning_fog),
|
||
"read_smog": finite(smog),
|
||
"read_coleman_liau": finite(coleman_liau),
|
||
"read_ari": finite(ari),
|
||
"read_words_per_sentence": finite(words_per_sentence),
|
||
}
|