53 lines
1.3 KiB
Python
53 lines
1.3 KiB
Python
"""Тесты чтения и валидации корпуса."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
from stylometry.io import (
|
|
DEFAULT_DATA_PATH,
|
|
compare_with_stats,
|
|
corpus_fingerprint,
|
|
read_corpus,
|
|
read_stats,
|
|
validate_corpus,
|
|
)
|
|
|
|
|
|
def test_read_corpus_shape() -> None:
|
|
frame = read_corpus()
|
|
assert len(frame) == 2005
|
|
assert frame["character"].nunique() == 34
|
|
assert {"line_id", "text", "character", "text_raw", "placeholder_count"} <= set(frame.columns)
|
|
assert frame["text"].str.len().gt(0).all()
|
|
|
|
|
|
def test_corpus_is_cp1251() -> None:
|
|
raw = DEFAULT_DATA_PATH.read_bytes()
|
|
assert raw.decode("cp1251").startswith("Реплика;Персонаж")
|
|
|
|
|
|
def test_validate_corpus() -> None:
|
|
frame = read_corpus()
|
|
stats = validate_corpus(frame)
|
|
assert stats["n_lines"] == 2005
|
|
assert stats["n_characters"] == 34
|
|
assert stats["n_duplicate_texts"] == 0
|
|
assert stats["n_words"] > 30_000
|
|
|
|
|
|
def test_compare_with_stats() -> None:
|
|
frame = read_corpus()
|
|
comparison = compare_with_stats(frame, read_stats())
|
|
assert not comparison.empty
|
|
assert (comparison["difference"] == 0).all()
|
|
|
|
|
|
def test_fingerprint_is_stable() -> None:
|
|
first = corpus_fingerprint()
|
|
second = corpus_fingerprint()
|
|
assert first == second
|
|
assert len(first) == 64
|