34 lines
951 B
Python
34 lines
951 B
Python
"""Тесты нормализации текста."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from stylometry.cleaning import (
|
|
clean_text,
|
|
count_placeholders,
|
|
normalize_placeholders,
|
|
normalize_unicode,
|
|
normalized_key,
|
|
)
|
|
|
|
|
|
def test_placeholders_are_replaced() -> None:
|
|
text = "Hello, <PLAYER>! Your <FARM> is nice."
|
|
assert normalize_placeholders(text) == "Hello, PLAYER! Your FARM is nice."
|
|
assert count_placeholders(text) == 2
|
|
|
|
|
|
def test_unknown_placeholder_becomes_entity() -> None:
|
|
assert normalize_placeholders("Look, <UNKNOWN>!") == "Look, ENTITY!"
|
|
|
|
|
|
def test_unicode_normalization() -> None:
|
|
assert normalize_unicode("Oh… really—yes") == "Oh... really-yes"
|
|
|
|
|
|
def test_whitespace_collapse() -> None:
|
|
assert clean_text(" Hello\n\n world ") == "Hello world"
|
|
|
|
|
|
def test_normalized_key_ignores_case_and_punctuation() -> None:
|
|
assert normalized_key("Oh, <PLAYER>!") == normalized_key("oh player")
|