Первичная версия: стилометрия реплик, LSA и протокол экспериментов
This commit is contained in:
175
lab1/stylometry/lexicons.py
Normal file
175
lab1/stylometry/lexicons.py
Normal file
@@ -0,0 +1,175 @@
|
||||
"""Закрытые классы слов и небольшие лексиконы для стилометрии.
|
||||
|
||||
Списки фиксированы заранее и не зависят от корпуса. Это важно для
|
||||
воспроизводимости: признаки не подбираются по тестовой выборке.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
ARTICLES = {"a", "an", "the"}
|
||||
|
||||
DETERMINERS = {
|
||||
"this", "that", "these", "those",
|
||||
"my", "your", "his", "her", "its", "our", "their",
|
||||
"some", "any", "no", "every", "each", "either", "neither",
|
||||
"both", "all", "few", "many", "much", "more", "most",
|
||||
"other", "another", "such", "what", "which", "whose",
|
||||
}
|
||||
|
||||
PREPOSITIONS = {
|
||||
"about", "above", "across", "after", "against", "along", "among",
|
||||
"around", "at", "before", "behind", "below", "beneath", "beside",
|
||||
"between", "beyond", "by", "despite", "down", "during", "except",
|
||||
"for", "from", "in", "inside", "into", "like", "near", "of", "off",
|
||||
"on", "onto", "out", "outside", "over", "past", "since", "through",
|
||||
"throughout", "to", "toward", "towards", "under", "until", "up",
|
||||
"upon", "with", "within", "without",
|
||||
}
|
||||
|
||||
COORDINATORS = {"and", "but", "or", "nor", "for", "so", "yet"}
|
||||
|
||||
SUBORDINATORS = {
|
||||
"because", "although", "though", "while", "whereas", "if", "unless",
|
||||
"since", "as", "than", "whether", "after", "before", "until", "when",
|
||||
"where", "once", "whenever", "wherever",
|
||||
}
|
||||
|
||||
WH_WORDS = {
|
||||
"who", "whom", "whose", "what", "which", "when", "where", "why", "how",
|
||||
}
|
||||
|
||||
PERSONAL_PRONOUNS = {
|
||||
"i", "me", "my", "mine", "myself",
|
||||
"you", "your", "yours", "yourself", "yourselves",
|
||||
"he", "him", "his", "himself",
|
||||
"she", "her", "hers", "herself",
|
||||
"it", "its", "itself",
|
||||
"we", "us", "our", "ours", "ourselves",
|
||||
"they", "them", "their", "theirs", "themselves",
|
||||
}
|
||||
|
||||
PRONOUN_CATEGORIES: dict[str, set[str]] = {
|
||||
"first_singular": {"i", "me", "my", "mine", "myself"},
|
||||
"first_plural": {"we", "us", "our", "ours", "ourselves"},
|
||||
"second": {"you", "your", "yours", "yourself", "yourselves"},
|
||||
"third_singular": {"he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself"},
|
||||
"third_plural": {"they", "them", "their", "theirs", "themselves"},
|
||||
"possessive": {"my", "mine", "your", "yours", "his", "her", "hers", "its", "our", "ours", "their", "theirs"},
|
||||
"reflexive": {"myself", "yourself", "yourselves", "himself", "herself", "itself", "ourselves", "themselves"},
|
||||
}
|
||||
|
||||
AUXILIARIES = {
|
||||
"am", "is", "are", "was", "were", "be", "been", "being",
|
||||
"have", "has", "had", "having",
|
||||
"do", "does", "did", "doing", "done",
|
||||
"'m", "'re", "'s", "'ve", "'ll", "'d",
|
||||
}
|
||||
|
||||
MODALS = {
|
||||
"can", "could", "may", "might", "must", "shall", "should", "will",
|
||||
"would", "ought", "need", "dare",
|
||||
}
|
||||
|
||||
NEGATIONS = {
|
||||
"not", "no", "never", "none", "nothing", "nobody", "nowhere",
|
||||
"neither", "nor", "n't", "cannot",
|
||||
"can't", "won't", "don't", "doesn't", "didn't", "isn't", "aren't",
|
||||
"wasn't", "weren't", "haven't", "hasn't", "hadn't", "couldn't",
|
||||
"wouldn't", "shouldn't", "mustn't", "ain't",
|
||||
}
|
||||
|
||||
CONTRACTIONS = {
|
||||
"i'm", "i've", "i'll", "i'd",
|
||||
"you're", "you've", "you'll", "you'd",
|
||||
"he's", "he'll", "he'd",
|
||||
"she's", "she'll", "she'd",
|
||||
"it's", "it'll",
|
||||
"we're", "we've", "we'll", "we'd",
|
||||
"they're", "they've", "they'll", "they'd",
|
||||
"that's", "there's", "what's", "who's", "where's", "how's",
|
||||
"can't", "won't", "don't", "doesn't", "didn't", "isn't", "aren't",
|
||||
"wasn't", "weren't", "haven't", "hasn't", "hadn't", "couldn't",
|
||||
"wouldn't", "shouldn't", "mustn't", "ain't",
|
||||
}
|
||||
|
||||
DISCOURSE_MARKERS = {
|
||||
"oh", "ah", "eh", "hey", "hi", "hello", "huh", "hmm", "um", "uh",
|
||||
"well", "so", "like", "just", "really", "actually", "anyway",
|
||||
"maybe", "perhaps", "please", "thanks", "thank", "sorry", "wow",
|
||||
"yeah", "yes", "okay", "ok", "right", "still", "even", "only",
|
||||
"quite", "rather", "pretty", "too", "very",
|
||||
}
|
||||
|
||||
INTERJECTIONS = {
|
||||
"oh", "ah", "eh", "hey", "hi", "hello", "huh", "hmm", "um", "uh",
|
||||
"wow", "yeah", "yes", "okay", "ok", "sigh", "giggle", "sniff",
|
||||
"heh", "haha", "mmm", "phew", "ugh", "ouch", "oops", "yay",
|
||||
"hooray", "bye", "goodbye",
|
||||
}
|
||||
|
||||
HEDGES = {
|
||||
"maybe", "perhaps", "probably", "possibly", "guess", "suppose",
|
||||
"think", "kind", "sort", "like", "somewhat", "rather", "fairly",
|
||||
"quite",
|
||||
}
|
||||
|
||||
INTENSIFIERS = {
|
||||
"very", "really", "so", "too", "pretty", "quite", "absolutely",
|
||||
"totally", "completely", "extremely", "incredibly", "terribly",
|
||||
"awfully", "highly",
|
||||
}
|
||||
|
||||
POLITENESS = {
|
||||
"please", "thanks", "thank", "sorry", "excuse", "pardon", "welcome",
|
||||
"kindly",
|
||||
}
|
||||
|
||||
# Объединённый список функциональных слов. Он используется и для
|
||||
# относительных частот, и для отбора функциональных слов в LSA.
|
||||
FUNCTION_WORDS: set[str] = set().union(
|
||||
ARTICLES,
|
||||
DETERMINERS,
|
||||
PREPOSITIONS,
|
||||
COORDINATORS,
|
||||
SUBORDINATORS,
|
||||
WH_WORDS,
|
||||
PERSONAL_PRONOUNS,
|
||||
AUXILIARIES,
|
||||
MODALS,
|
||||
NEGATIONS,
|
||||
CONTRACTIONS,
|
||||
)
|
||||
|
||||
FUNCTION_WORD_CATEGORIES: dict[str, set[str]] = {
|
||||
"articles": ARTICLES,
|
||||
"determiners": DETERMINERS,
|
||||
"prepositions": PREPOSITIONS,
|
||||
"coordinators": COORDINATORS,
|
||||
"subordinators": SUBORDINATORS,
|
||||
"wh_words": WH_WORDS,
|
||||
"pronouns": PERSONAL_PRONOUNS,
|
||||
"auxiliaries": AUXILIARIES,
|
||||
"modals": MODALS,
|
||||
"negations": NEGATIONS,
|
||||
"contractions": CONTRACTIONS,
|
||||
"discourse_markers": DISCOURSE_MARKERS,
|
||||
}
|
||||
|
||||
# Запасной список стоп-слов, если NLTK-ресурс stopwords недоступен.
|
||||
STOPWORD_FALLBACK = set().union(
|
||||
ARTICLES,
|
||||
DETERMINERS,
|
||||
PREPOSITIONS,
|
||||
COORDINATORS,
|
||||
SUBORDINATORS,
|
||||
WH_WORDS,
|
||||
PERSONAL_PRONOUNS,
|
||||
AUXILIARIES,
|
||||
MODALS,
|
||||
NEGATIONS,
|
||||
{"also", "just", "then", "there", "here", "now", "very", "too", "only"},
|
||||
)
|
||||
|
||||
# Слова, которые в NLTK помечаются как IN, но синтаксически являются
|
||||
# подчинительными союзами. Используются при переходе к Universal POS.
|
||||
SUBORDINATING_IN = SUBORDINATORS
|
||||
Reference in New Issue
Block a user