Первичная версия: стилометрия реплик, LSA и протокол экспериментов

lab1/tests/test_cleaning.py

@@ -0,0 +1,33 @@
+"""Тесты нормализации текста."""
+
+from __future__ import annotations
+
+from stylometry.cleaning import (
+    clean_text,
+    count_placeholders,
+    normalize_placeholders,
+    normalize_unicode,
+    normalized_key,
+)
+
+
+def test_placeholders_are_replaced() -> None:
+    text = "Hello, <PLAYER>! Your <FARM> is nice."
+    assert normalize_placeholders(text) == "Hello, PLAYER! Your FARM is nice."
+    assert count_placeholders(text) == 2
+
+
+def test_unknown_placeholder_becomes_entity() -> None:
+    assert normalize_placeholders("Look, <UNKNOWN>!") == "Look, ENTITY!"
+
+
+def test_unicode_normalization() -> None:
+    assert normalize_unicode("Oh… really—yes") == "Oh... really-yes"
+
+
+def test_whitespace_collapse() -> None:
+    assert clean_text("  Hello\n\n  world  ") == "Hello world"
+
+
+def test_normalized_key_ignores_case_and_punctuation() -> None:
+    assert normalized_key("Oh, <PLAYER>!") == normalized_key("oh player")