46 lines
1.4 KiB
Python
46 lines
1.4 KiB
Python
"""Тесты TF-IDF и LSA."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
|
|
from stylometry.datasets import load_feature_frame
|
|
from stylometry.lsa import (
|
|
make_char_ngram_vectorizer,
|
|
make_lsa_vectorizer,
|
|
make_pos_ngram_vectorizer,
|
|
make_text_vectorizer,
|
|
)
|
|
|
|
|
|
def test_lsa_word_shape_and_variance() -> None:
|
|
frame = load_feature_frame(verbose=False).head(100)
|
|
vectorizer = make_lsa_vectorizer("word", n_components=10)
|
|
matrix = vectorizer.fit_transform(frame)
|
|
assert matrix.shape == (100, 10)
|
|
assert np.isfinite(matrix).all()
|
|
assert 0 <= vectorizer.cumulative_explained_variance_[-1] <= 1.0
|
|
assert len(vectorizer.top_terms(5)) == 10
|
|
|
|
|
|
def test_lsa_pos_and_char() -> None:
|
|
frame = load_feature_frame(verbose=False).head(100)
|
|
for variant in ("pos", "char"):
|
|
vectorizer = make_lsa_vectorizer(variant, n_components=5)
|
|
matrix = vectorizer.fit_transform(frame)
|
|
assert matrix.shape[0] == 100
|
|
assert matrix.shape[1] == 5
|
|
assert np.isfinite(matrix).all()
|
|
|
|
|
|
def test_text_vectorizers() -> None:
|
|
frame = load_feature_frame(verbose=False).head(50)
|
|
for vectorizer in (
|
|
make_text_vectorizer("word"),
|
|
make_pos_ngram_vectorizer(),
|
|
make_char_ngram_vectorizer(),
|
|
):
|
|
matrix = vectorizer.fit_transform(frame)
|
|
assert matrix.shape[0] == 50
|
|
assert matrix.shape[1] > 0
|