87 lines
3.1 KiB
Python
87 lines
3.1 KiB
Python
"""Подготовка таблицы реплик с токенами, тегами и текстами для LSA."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from pathlib import Path
|
||
|
||
import pandas as pd
|
||
|
||
from .io import DEFAULT_DATA_PATH, corpus_fingerprint, read_corpus
|
||
from .tokenization import Tokenizer
|
||
|
||
DEFAULT_CACHE_PATH = Path(__file__).resolve().parents[1] / "data" / "processed" / "tokenized.pkl"
|
||
|
||
REQUIRED_COLUMNS = {
|
||
"line_id",
|
||
"character",
|
||
"text",
|
||
"tokens",
|
||
"ptb",
|
||
"upos",
|
||
"sentences",
|
||
"pos_text",
|
||
"fw_text",
|
||
"content_text",
|
||
}
|
||
|
||
|
||
def build_feature_frame(
|
||
data_path: str | Path | None = None,
|
||
tokenizer: Tokenizer | None = None,
|
||
verbose: bool = True,
|
||
) -> pd.DataFrame:
|
||
"""Читает корпус и добавляет результаты токенизации и разметки."""
|
||
data_path = Path(data_path) if data_path is not None else DEFAULT_DATA_PATH
|
||
frame = read_corpus(data_path)
|
||
tokenizer = tokenizer or Tokenizer(use_nltk=True)
|
||
features = tokenizer.analyze_frame(frame, verbose=verbose)
|
||
features.attrs["corpus_path"] = str(data_path)
|
||
features.attrs["corpus_sha256"] = corpus_fingerprint(data_path)
|
||
features.attrs["nltk_used"] = tokenizer.use_nltk
|
||
features.attrs["pos_available"] = tokenizer.pos_available
|
||
return features
|
||
|
||
|
||
def save_feature_frame(frame: pd.DataFrame, path: str | Path) -> Path:
|
||
path = Path(path)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
frame.to_pickle(path)
|
||
return path
|
||
|
||
|
||
def load_feature_frame(
|
||
data_path: str | Path | None = None,
|
||
cache_path: str | Path | None = None,
|
||
force: bool = False,
|
||
verbose: bool = True,
|
||
) -> pd.DataFrame:
|
||
"""Загружает кэш токенизации или строит его заново.
|
||
|
||
Кэш проверяется по SHA-256 исходного файла. Если файл корпуса изменился,
|
||
токенизация выполняется повторно.
|
||
"""
|
||
data_path = Path(data_path) if data_path is not None else DEFAULT_DATA_PATH
|
||
cache_path = Path(cache_path) if cache_path is not None else DEFAULT_CACHE_PATH
|
||
|
||
if cache_path.exists() and not force:
|
||
try:
|
||
frame = pd.read_pickle(cache_path)
|
||
if REQUIRED_COLUMNS <= set(frame.columns):
|
||
cached_hash = frame.attrs.get("corpus_sha256")
|
||
current_hash = corpus_fingerprint(data_path)
|
||
if cached_hash == current_hash:
|
||
if verbose:
|
||
print(f"[datasets] загружен кэш: {cache_path}")
|
||
return frame
|
||
if verbose:
|
||
print("[datasets] корпус изменился, пересобираю признаки.")
|
||
except Exception as error:
|
||
if verbose:
|
||
print(f"[datasets] не удалось прочитать кэш ({error}), пересобираю.")
|
||
|
||
frame = build_feature_frame(data_path, verbose=verbose)
|
||
save_feature_frame(frame, cache_path)
|
||
if verbose:
|
||
print(f"[datasets] кэш сохранён: {cache_path}")
|
||
return frame
|