Первичная версия: стилометрия реплик, LSA и протокол экспериментов

lab1/tests/test_tokenization.py

@@ -0,0 +1,40 @@
+"""Тесты NLTK-токенизации и частеречной разметки."""
+
+from __future__ import annotations
+
+from stylometry.tokenization import Tokenizer, penn_to_upos
+
+
+def test_penn_to_upos() -> None:
+    assert penn_to_upos("the", "DT") == "DET"
+    assert penn_to_upos("run", "VB") == "VERB"
+    assert penn_to_upos("is", "VBZ") == "AUX"
+    assert penn_to_upos("because", "IN") == "SCONJ"
+    assert penn_to_upos("quickly", "RB") == "ADV"
+
+
+def test_analyze_line() -> None:
+    tokenizer = Tokenizer(use_nltk=True)
+    analyzed = tokenizer.analyze("Oh... hi. I don't know, Mr. Qi.")
+    assert analyzed.tokens
+    assert len(analyzed.tokens) == len(analyzed.upos) == len(analyzed.ptb)
+    assert analyzed.sentences
+    assert analyzed.pos_text
+    assert analyzed.fw_text
+    assert analyzed.content_text
+
+
+def test_placeholders_are_tagged_as_words() -> None:
+    tokenizer = Tokenizer(use_nltk=True)
+    analyzed = tokenizer.analyze("Hello, PLAYER! You are nice.")
+    assert "PLAYER" in analyzed.tokens
+    assert len(analyzed.upos) == len(analyzed.tokens)
+
+
+def test_fallback_regex_tokenizer() -> None:
+    tokenizer = Tokenizer(use_nltk=False)
+    analyzed = tokenizer.analyze("Hello, world! It's fine.")
+    assert analyzed.tokens == ["Hello", ",", "world", "!", "It's", "fine", "."]
+    assert len(analyzed.upos) == len(analyzed.tokens)
+    assert analyzed.upos[0] == "X"
+    assert "PUNCT" in analyzed.upos