Mermutef/NLP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Добавлены параметры моделей, веса блоков и руководство по настройке

Browse Source
This commit is contained in:
Vladislav Kopnin
2026-10-25 16:08:00 +03:00
parent 10e2444d23
commit 5a4a19e0df
7 changed files with 537 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
lab1/stylometry/cli.py
View File

@@ -373,13 +373,24 @@ def cmd_run(args: argparse.Namespace) -> int:
)
overrides = dict(config_data.get("features", {}))
for key in ("char_ngram_range", "pos_ngram_range"):
for key in ("char_ngram_range", "pos_ngram_range", "groups"):
if key in overrides and isinstance(overrides[key], list):
overrides[key] = tuple(overrides[key])
if "lsa_variants" in overrides and isinstance(overrides["lsa_variants"], list):
overrides["lsa_variants"] = tuple(overrides["lsa_variants"])
feature_config = _feature_config(preset, overrides)
pipeline = build_pipeline(feature_config, model)
model_params = dict(config_data.get("model_params", {}))
if args.model_params:
try:
extra_params = json.loads(args.model_params)
except json.JSONDecodeError as error:
raise ValueError(f"Некорректный JSON в --model-params: {error}") from error
if not isinstance(extra_params, dict):
raise ValueError("--model-params должен быть JSON-объектом.")
model_params.update(extra_params)
pipeline = build_pipeline(feature_config, model, **model_params)
run_config = {
"preset": preset,
"model": model,
@@ -390,6 +401,7 @@ def cmd_run(args: argparse.Namespace) -> int:
"n_classes": int(len(np.unique(y))),
"corpus_sha256": frame.attrs.get("corpus_sha256", ""),
"feature_config": dict(feature_config.__dict__),
"model_params": model_params,
"package_versions": _package_versions(),
}
@@ -492,6 +504,7 @@ def cmd_ablate(args: argparse.Namespace) -> int:
lsa_components=base.lsa_components,
stylo_scaler=base.stylo_scaler,
stylo_normalize=base.stylo_normalize,
block_weights=dict(base.block_weights),
include_stylo=True,
include_lsa=base.include_lsa,
char_max_features=base.char_max_features,
@@ -772,6 +785,11 @@ def build_parser() -> argparse.ArgumentParser:
run.add_argument("--folds", type=int, default=None)
run.add_argument("--repeats", type=int, default=None)
run.add_argument("--seed", type=int, default=None)
run.add_argument(
"--model-params",
default=None,
help='JSON с параметрами модели, например \'{"C": 0.3, "class_weight": "balanced"}\'',
)
run.add_argument("--nested", action="store_true", help="Вложенная кросс-валидация")
run.set_defaults(func=cmd_run)

41
lab1/stylometry/pipelines.py
View File

@@ -45,6 +45,9 @@ class FeatureConfig:
lsa_components: int = 100
stylo_scaler: str = "standard" # standard | minmax | none
stylo_normalize: bool = True
# Веса блоков в FeatureUnion: stylo, char, pos, lsa_word, lsa_pos и т.д.
# Пустой словарь означает равный вес блоков.
block_weights: dict[str, float] = field(default_factory=dict)
include_stylo: bool = True
include_lsa: bool = True
char_max_features: int = 5000
@@ -150,7 +153,18 @@ def build_feature_union(config: FeatureConfig) -> Any:
if len(branches) == 1:
return branches[0][1]
return FeatureUnion(branches)
union_kwargs: dict[str, Any] = {}
if config.block_weights:
branch_names = {name for name, _ in branches}
unknown = set(config.block_weights) - branch_names
if unknown:
raise ValueError(
f"Неизвестные блоки в block_weights: {sorted(unknown)}. "
f"Доступны: {sorted(branch_names)}"
)
union_kwargs["transformer_weights"] = dict(config.block_weights)
return FeatureUnion(branches, **union_kwargs)
def make_classifier(name: str, **params: Any) -> Any:
@@ -159,19 +173,24 @@ def make_classifier(name: str, **params: Any) -> Any:
strategy = params.pop("strategy", "most_frequent")
return DummyClassifier(strategy=strategy)
if name == "multinomial_nb":
return MultinomialNB(alpha=params.pop("alpha", 1.0))
return MultinomialNB(
alpha=params.pop("alpha", 1.0),
fit_prior=params.pop("fit_prior", True),
)
if name == "logreg":
return LogisticRegression(
C=params.pop("C", 1.0),
solver="lbfgs",
max_iter=5000,
class_weight="balanced",
solver=params.pop("solver", "lbfgs"),
max_iter=params.pop("max_iter", 5000),
class_weight=params.pop("class_weight", "balanced"),
)
if name == "linear_svc":
return LinearSVC(
C=params.pop("C", 1.0),
class_weight="balanced",
max_iter=20_000,
loss=params.pop("loss", "squared_hinge"),
tol=params.pop("tol", 1e-4),
class_weight=params.pop("class_weight", "balanced"),
max_iter=params.pop("max_iter", 20_000),
dual="auto",
)
if name == "random_forest":
@@ -179,7 +198,9 @@ def make_classifier(name: str, **params: Any) -> Any:
n_estimators=params.pop("n_estimators", 400),
max_depth=params.pop("max_depth", None),
min_samples_leaf=params.pop("min_samples_leaf", 1),
class_weight="balanced_subsample",
min_samples_split=params.pop("min_samples_split", 2),
max_features=params.pop("max_features", "sqrt"),
class_weight=params.pop("class_weight", "balanced_subsample"),
n_jobs=-1,
random_state=params.pop("random_state", 42),
)
@@ -188,7 +209,9 @@ def make_classifier(name: str, **params: Any) -> Any:
n_estimators=params.pop("n_estimators", 400),
max_depth=params.pop("max_depth", None),
min_samples_leaf=params.pop("min_samples_leaf", 1),
class_weight="balanced",
min_samples_split=params.pop("min_samples_split", 2),
max_features=params.pop("max_features", "sqrt"),
class_weight=params.pop("class_weight", "balanced"),
n_jobs=-1,
random_state=params.pop("random_state", 42),
)