41 lines
1.4 KiB
Python
41 lines
1.4 KiB
Python
"""Тесты NLTK-токенизации и частеречной разметки."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from stylometry.tokenization import Tokenizer, penn_to_upos
|
|
|
|
|
|
def test_penn_to_upos() -> None:
|
|
assert penn_to_upos("the", "DT") == "DET"
|
|
assert penn_to_upos("run", "VB") == "VERB"
|
|
assert penn_to_upos("is", "VBZ") == "AUX"
|
|
assert penn_to_upos("because", "IN") == "SCONJ"
|
|
assert penn_to_upos("quickly", "RB") == "ADV"
|
|
|
|
|
|
def test_analyze_line() -> None:
|
|
tokenizer = Tokenizer(use_nltk=True)
|
|
analyzed = tokenizer.analyze("Oh... hi. I don't know, Mr. Qi.")
|
|
assert analyzed.tokens
|
|
assert len(analyzed.tokens) == len(analyzed.upos) == len(analyzed.ptb)
|
|
assert analyzed.sentences
|
|
assert analyzed.pos_text
|
|
assert analyzed.fw_text
|
|
assert analyzed.content_text
|
|
|
|
|
|
def test_placeholders_are_tagged_as_words() -> None:
|
|
tokenizer = Tokenizer(use_nltk=True)
|
|
analyzed = tokenizer.analyze("Hello, PLAYER! You are nice.")
|
|
assert "PLAYER" in analyzed.tokens
|
|
assert len(analyzed.upos) == len(analyzed.tokens)
|
|
|
|
|
|
def test_fallback_regex_tokenizer() -> None:
|
|
tokenizer = Tokenizer(use_nltk=False)
|
|
analyzed = tokenizer.analyze("Hello, world! It's fine.")
|
|
assert analyzed.tokens == ["Hello", ",", "world", "!", "It's", "fine", "."]
|
|
assert len(analyzed.upos) == len(analyzed.tokens)
|
|
assert analyzed.upos[0] == "X"
|
|
assert "PUNCT" in analyzed.upos
|