NLP/lab1/stylometry/lexicons.py

"""Закрытые классы слов и небольшие лексиконы для стилометрии.

Списки фиксированы заранее и не зависят от корпуса. Это важно для
воспроизводимости: признаки не подбираются по тестовой выборке.
"""

from __future__ import annotations

ARTICLES = {"a", "an", "the"}

DETERMINERS = {
    "this", "that", "these", "those",
    "my", "your", "his", "her", "its", "our", "their",
    "some", "any", "no", "every", "each", "either", "neither",
    "both", "all", "few", "many", "much", "more", "most",
    "other", "another", "such", "what", "which", "whose",
}

PREPOSITIONS = {
    "about", "above", "across", "after", "against", "along", "among",
    "around", "at", "before", "behind", "below", "beneath", "beside",
    "between", "beyond", "by", "despite", "down", "during", "except",
    "for", "from", "in", "inside", "into", "like", "near", "of", "off",
    "on", "onto", "out", "outside", "over", "past", "since", "through",
    "throughout", "to", "toward", "towards", "under", "until", "up",
    "upon", "with", "within", "without",
}

COORDINATORS = {"and", "but", "or", "nor", "for", "so", "yet"}

SUBORDINATORS = {
    "because", "although", "though", "while", "whereas", "if", "unless",
    "since", "as", "than", "whether", "after", "before", "until", "when",
    "where", "once", "whenever", "wherever",
}

WH_WORDS = {
    "who", "whom", "whose", "what", "which", "when", "where", "why", "how",
}

PERSONAL_PRONOUNS = {
    "i", "me", "my", "mine", "myself",
    "you", "your", "yours", "yourself", "yourselves",
    "he", "him", "his", "himself",
    "she", "her", "hers", "herself",
    "it", "its", "itself",
    "we", "us", "our", "ours", "ourselves",
    "they", "them", "their", "theirs", "themselves",
}

PRONOUN_CATEGORIES: dict[str, set[str]] = {
    "first_singular": {"i", "me", "my", "mine", "myself"},
    "first_plural": {"we", "us", "our", "ours", "ourselves"},
    "second": {"you", "your", "yours", "yourself", "yourselves"},
    "third_singular": {"he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself"},
    "third_plural": {"they", "them", "their", "theirs", "themselves"},
    "possessive": {"my", "mine", "your", "yours", "his", "her", "hers", "its", "our", "ours", "their", "theirs"},
    "reflexive": {"myself", "yourself", "yourselves", "himself", "herself", "itself", "ourselves", "themselves"},
}

AUXILIARIES = {
    "am", "is", "are", "was", "were", "be", "been", "being",
    "have", "has", "had", "having",
    "do", "does", "did", "doing", "done",
    "'m", "'re", "'s", "'ve", "'ll", "'d",
}

MODALS = {
    "can", "could", "may", "might", "must", "shall", "should", "will",
    "would", "ought", "need", "dare",
}

NEGATIONS = {
    "not", "no", "never", "none", "nothing", "nobody", "nowhere",
    "neither", "nor", "n't", "cannot",
    "can't", "won't", "don't", "doesn't", "didn't", "isn't", "aren't",
    "wasn't", "weren't", "haven't", "hasn't", "hadn't", "couldn't",
    "wouldn't", "shouldn't", "mustn't", "ain't",
}

CONTRACTIONS = {
    "i'm", "i've", "i'll", "i'd",
    "you're", "you've", "you'll", "you'd",
    "he's", "he'll", "he'd",
    "she's", "she'll", "she'd",
    "it's", "it'll",
    "we're", "we've", "we'll", "we'd",
    "they're", "they've", "they'll", "they'd",
    "that's", "there's", "what's", "who's", "where's", "how's",
    "can't", "won't", "don't", "doesn't", "didn't", "isn't", "aren't",
    "wasn't", "weren't", "haven't", "hasn't", "hadn't", "couldn't",
    "wouldn't", "shouldn't", "mustn't", "ain't",
}

DISCOURSE_MARKERS = {
    "oh", "ah", "eh", "hey", "hi", "hello", "huh", "hmm", "um", "uh",
    "well", "so", "like", "just", "really", "actually", "anyway",
    "maybe", "perhaps", "please", "thanks", "thank", "sorry", "wow",
    "yeah", "yes", "okay", "ok", "right", "still", "even", "only",
    "quite", "rather", "pretty", "too", "very",
}

INTERJECTIONS = {
    "oh", "ah", "eh", "hey", "hi", "hello", "huh", "hmm", "um", "uh",
    "wow", "yeah", "yes", "okay", "ok", "sigh", "giggle", "sniff",
    "heh", "haha", "mmm", "phew", "ugh", "ouch", "oops", "yay",
    "hooray", "bye", "goodbye",
}

HEDGES = {
    "maybe", "perhaps", "probably", "possibly", "guess", "suppose",
    "think", "kind", "sort", "like", "somewhat", "rather", "fairly",
    "quite",
}

INTENSIFIERS = {
    "very", "really", "so", "too", "pretty", "quite", "absolutely",
    "totally", "completely", "extremely", "incredibly", "terribly",
    "awfully", "highly",
}

POLITENESS = {
    "please", "thanks", "thank", "sorry", "excuse", "pardon", "welcome",
    "kindly",
}

# Объединённый список функциональных слов. Он используется и для
# относительных частот, и для отбора функциональных слов в LSA.
FUNCTION_WORDS: set[str] = set().union(
    ARTICLES,
    DETERMINERS,
    PREPOSITIONS,
    COORDINATORS,
    SUBORDINATORS,
    WH_WORDS,
    PERSONAL_PRONOUNS,
    AUXILIARIES,
    MODALS,
    NEGATIONS,
    CONTRACTIONS,
)

FUNCTION_WORD_CATEGORIES: dict[str, set[str]] = {
    "articles": ARTICLES,
    "determiners": DETERMINERS,
    "prepositions": PREPOSITIONS,
    "coordinators": COORDINATORS,
    "subordinators": SUBORDINATORS,
    "wh_words": WH_WORDS,
    "pronouns": PERSONAL_PRONOUNS,
    "auxiliaries": AUXILIARIES,
    "modals": MODALS,
    "negations": NEGATIONS,
    "contractions": CONTRACTIONS,
    "discourse_markers": DISCOURSE_MARKERS,
}

# Запасной список стоп-слов, если NLTK-ресурс stopwords недоступен.
STOPWORD_FALLBACK = set().union(
    ARTICLES,
    DETERMINERS,
    PREPOSITIONS,
    COORDINATORS,
    SUBORDINATORS,
    WH_WORDS,
    PERSONAL_PRONOUNS,
    AUXILIARIES,
    MODALS,
    NEGATIONS,
    {"also", "just", "then", "there", "here", "now", "very", "too", "only"},
)

# Слова, которые в NLTK помечаются как IN, но синтаксически являются
# подчинительными союзами. Используются при переходе к Universal POS.
SUBORDINATING_IN = SUBORDINATORS