"""Тесты NLTK-токенизации и частеречной разметки."""

from __future__ import annotations

from stylometry.tokenization import Tokenizer, penn_to_upos


def test_penn_to_upos() -> None:
    assert penn_to_upos("the", "DT") == "DET"
    assert penn_to_upos("run", "VB") == "VERB"
    assert penn_to_upos("is", "VBZ") == "AUX"
    assert penn_to_upos("because", "IN") == "SCONJ"
    assert penn_to_upos("quickly", "RB") == "ADV"


def test_analyze_line() -> None:
    tokenizer = Tokenizer(use_nltk=True)
    analyzed = tokenizer.analyze("Oh... hi. I don't know, Mr. Qi.")
    assert analyzed.tokens
    assert len(analyzed.tokens) == len(analyzed.upos) == len(analyzed.ptb)
    assert analyzed.sentences
    assert analyzed.pos_text
    assert analyzed.fw_text
    assert analyzed.content_text


def test_placeholders_are_tagged_as_words() -> None:
    tokenizer = Tokenizer(use_nltk=True)
    analyzed = tokenizer.analyze("Hello, PLAYER! You are nice.")
    assert "PLAYER" in analyzed.tokens
    assert len(analyzed.upos) == len(analyzed.tokens)


def test_fallback_regex_tokenizer() -> None:
    tokenizer = Tokenizer(use_nltk=False)
    analyzed = tokenizer.analyze("Hello, world! It's fine.")
    assert analyzed.tokens == ["Hello", ",", "world", "!", "It's", "fine", "."]
    assert len(analyzed.upos) == len(analyzed.tokens)
    assert analyzed.upos[0] == "X"
    assert "PUNCT" in analyzed.upos