scripts/ablation/hyperparameters.py - master - Mihaham/CNN-NEAT

scripts/ablation/hyperparameters.py
505 строк18 KB
MihahamYT
All changes
18 июл 2026, 18:53
18 июл 2026, 18:53e49dccd
"""Hyperparameter registry for CNN-NEAT architecture ablation studies.

This file is the **single source of truth** for all ablation configurations.
Both ``02_arch_ablation.py`` and ``03_final_report.py`` import from here.

Usage::

    from scripts.ablation.hyperparameters import BASELINE_CFG, ABLATION_GRID
    cfg = ABLATION_GRID["activation_relu"]

Design principle
----------------
One condition changes at a time relative to BASELINE_CFG.  This makes it
easy to attribute performance differences to specific architectural choices.

After running all configs and analysing results, add a ``best_combined`` entry
that combines the winning settings from Phase 1.

The BASELINE_CFG intentionally uses:
  - No activation (purely linear conv graph, cheapest forward pass)
  - No edge weights
  - Fixed channels (3)
  - Moderate population and generation counts for a production-scale run

Override ``num_generations`` via ``--generations N`` in 02_arch_ablation.py
for quick smoke tests (e.g. N=5) without editing this file.
"""

from __future__ import annotations

from dataclasses import replace

from cnn_neat.experiment import ExperimentConfig
from cnn_neat.paths import repo_root

_ABLATION_SAVE_ROOT = str(repo_root() / "runs" / "ablation")

# ---------------------------------------------------------------------------
# Baseline configuration
# ---------------------------------------------------------------------------

BASELINE_CFG = ExperimentConfig(
    # Task
    seed=42,
    positive_class=0,  # airplane vs. rest

    # Network topology
    activation=None,
    input_channels=3,
    allowed_channels=None,          # library default: (3, 8, 16)
    allow_variable_channels=False,
    edge_weights=False,

    # Evolution
    population_size=100,
    num_generations=50,
    elite_percent=0.2,

    # Elite refinement (off by default in ablation)
    elite_refinement_enabled=False,

    # Fitness
    fitness_metric="balanced_accuracy",
    fitness_val_size=25000,
    eval_batch_size=25000,          # full val set — VRAM controller will shrink if needed
    adaptive_eval_batch_size=True,
    vram_usage_threshold=0.80,
    eval_batch_size_min=256,

    # Periodic GD (off for Phase 1; separate config tests it)
    periodic_gd_enabled=False,

    # Artifacts
    save_format="archive",
    save_run_path=_ABLATION_SAVE_ROOT + "/baseline",
)


# ---------------------------------------------------------------------------
# Ablation grid — one condition changed per entry
# ---------------------------------------------------------------------------

ABLATION_GRID: dict[str, ExperimentConfig] = {

    # ── Phase 1: activation functions ──────────────────────────────────────
    "baseline": BASELINE_CFG,

    "activation_relu": replace(
        BASELINE_CFG,
        activation="relu",
        save_run_path=_ABLATION_SAVE_ROOT + "/activation_relu",
    ),
    "activation_gelu": replace(
        BASELINE_CFG,
        activation="gelu",
        save_run_path=_ABLATION_SAVE_ROOT + "/activation_gelu",
    ),
    "activation_tanh": replace(
        BASELINE_CFG,
        activation="tanh",
        save_run_path=_ABLATION_SAVE_ROOT + "/activation_tanh",
    ),
    "activation_leaky_relu": replace(
        BASELINE_CFG,
        activation="leaky_relu",
        save_run_path=_ABLATION_SAVE_ROOT + "/activation_leaky_relu",
    ),

    # ── Phase 1: readout modes ──────────────────────────────────────────────
    "readout_mean": replace(
        BASELINE_CFG,
        output_readout="mean",
        save_run_path=_ABLATION_SAVE_ROOT + "/readout_mean",
    ),

    # ── Phase 2: edge weights ───────────────────────────────────────────────
    "edge_weights": replace(
        BASELINE_CFG,
        edge_weights=True,
        save_run_path=_ABLATION_SAVE_ROOT + "/edge_weights",
    ),

    # ── Phase 2: variable channels ──────────────────────────────────────────
    "variable_channels": replace(
        BASELINE_CFG,
        allow_variable_channels=True,
        save_run_path=_ABLATION_SAVE_ROOT + "/variable_channels",
    ),
    "channels_8_16_32": replace(
        BASELINE_CFG,
        allow_variable_channels=True,
        allowed_channels=(8, 16, 32),
        save_run_path=_ABLATION_SAVE_ROOT + "/channels_8_16_32",
    ),
    "channels_16_32_64": replace(
        BASELINE_CFG,
        allow_variable_channels=True,
        allowed_channels=(16, 32, 64),
        save_run_path=_ABLATION_SAVE_ROOT + "/channels_16_32_64",
    ),

    # ── Phase 3: periodic gradient descent ─────────────────────────────────
    # Run these AFTER Phase 1+2 winner is identified (update best_combined).
    "periodic_gd_every5": replace(
        BASELINE_CFG,
        periodic_gd_enabled=True,
        periodic_gd_every=5,
        periodic_gd_epochs=3,
        periodic_gd_batch_size=256,
        save_run_path=_ABLATION_SAVE_ROOT + "/periodic_gd_every5",
    ),
    "periodic_gd_every10": replace(
        BASELINE_CFG,
        periodic_gd_enabled=True,
        periodic_gd_every=10,
        periodic_gd_epochs=5,
        periodic_gd_batch_size=256,
        save_run_path=_ABLATION_SAVE_ROOT + "/periodic_gd_every10",
    ),

    # ── Combined winner placeholder (fill in after Phase 1/2 analysis) ─────
    # "best_combined": replace(
    #     BASELINE_CFG,
    #     activation="relu",           # ← replace with Phase-1 winner
    #     edge_weights=False,          # ← replace with Phase-2 winner
    #     allow_variable_channels=True,
    #     allowed_channels=(8, 16, 32),
    #     periodic_gd_enabled=True,
    #     periodic_gd_every=5,
    #     save_run_path=_ABLATION_SAVE_ROOT + "/best_combined",
    # ),
}

# Phase groupings — used by 02_arch_ablation.py ``--phase`` flag
PHASE_GROUPS: dict[str, list[str]] = {
    "1": ["baseline", "activation_relu", "activation_gelu", "activation_tanh",
          "activation_leaky_relu", "readout_mean"],
    "2": ["edge_weights", "variable_channels", "channels_8_16_32", "channels_16_32_64"],
    "3": ["periodic_gd_every5", "periodic_gd_every10"],
    "all": list(ABLATION_GRID.keys()),
}


# ===========================================================================
# Building blocks for the structured sequential study (05_structured_study.py)
# ===========================================================================

# All activation options to sweep in Phase A (None = linear baseline)
ALL_ACTIVATIONS: list[str | None] = [None, "relu", "gelu", "tanh", "leaky_relu"]
ACTIVATION_LABELS: dict[str | None, str] = {
    None: "linear",
    "relu": "relu",
    "gelu": "gelu",
    "tanh": "tanh",
    "leaky_relu": "leaky_relu",
}

# Both readout (head) modes to compare against each activation
ALL_READOUTS: list[str] = ["mean_minus_median", "mean"]
READOUT_LABELS: dict[str, str] = {
    "mean_minus_median": "mmd",
    "mean": "mean",
}

# Channel-width configurations for Phase B/D.
# var_default: variable hidden channels {3,8,16} × edge_weights {off, on}
# fixed_3: fixed C=3 everywhere × edge_weights on only (4 extra runs at top_n=4)
CHANNEL_SETS: dict[str, tuple[int, ...] | None] = {
    "var_default": None,  # library default set {3, 8, 16}
    "fixed_3": None,      # allow_variable_channels=False — all nodes C=3
}
CHANNEL_SET_VARIABLE: dict[str, bool] = {
    "var_default": True,
    "fixed_3": False,
}
# edge_weights values per channel set (Phase B/D grid is not a full Cartesian product)
CHANNEL_SET_EDGE_WEIGHTS: dict[str, tuple[bool, ...]] = {
    "var_default": (False, True),
    "fixed_3": (True,),
}


def make_phase_a_grid(
    *,
    base_cfg: ExperimentConfig = BASELINE_CFG,
    with_gd: bool = False,
    save_root: str | None = None,
    gd_profile: str | None = None,
    gd_every: int = 5,
    gd_epochs: int = 3,
    gd_batch: int = 256,
    gd_max_batches: int | None = None,
    gd_cv_enabled: bool = False,
    gd_cv_folds: int = 5,
) -> dict[str, "ExperimentConfig"]:
    """Build the Phase-A grid: all activations × all readout modes.

    Parameters
    ----------
    with_gd:
        Add periodic gradient descent to every config.
    base_cfg:
        Starting point for every grid entry (default: BASELINE_CFG).
    save_root:
        Override the base save directory (default: runs/ablation/study).
    """
    root = save_root or str(repo_root() / "runs" / "ablation" / "study")
    if with_gd:
        suffix = f"_gd_{gd_profile}" if gd_profile else "_gd"
    else:
        suffix = ""
    gd_kwargs: dict = {}
    if with_gd:
        gd_kwargs = dict(
            periodic_gd_enabled=True,
            periodic_gd_every=gd_every,
            periodic_gd_epochs=gd_epochs,
            periodic_gd_batch_size=gd_batch,
            periodic_gd_max_batches_per_epoch=gd_max_batches,
            periodic_gd_cv_enabled=gd_cv_enabled,
            periodic_gd_cv_folds=gd_cv_folds,
        )

    grid: dict[str, ExperimentConfig] = {}
    for act in ALL_ACTIVATIONS:
        act_label = ACTIVATION_LABELS[act]
        for readout in ALL_READOUTS:
            ro_label = "mmd" if readout == "mean_minus_median" else "mean"
            key = f"{act_label}_{ro_label}{suffix}"
            grid[key] = replace(
                base_cfg,
                activation=act,
                output_readout=readout,
                save_run_path=f"{root}/{key}",
                **gd_kwargs,
            )
    return grid


def make_phase_b_grid(
    top_activations: list[str | None],
    *,
    activation_readouts: dict[str | None, str] | None = None,
    base_cfg: ExperimentConfig = BASELINE_CFG,
    with_gd: bool = False,
    channel_set_names: list[str] | None = None,
    save_root: str | None = None,
    gd_profile: str | None = None,
    gd_every: int = 5,
    gd_epochs: int = 3,
    gd_batch: int = 256,
    gd_max_batches: int | None = None,
    gd_cv_enabled: bool = False,
    gd_cv_folds: int = 5,
) -> dict[str, "ExperimentConfig"]:
    """Build the Phase-B grid: top-N activations × channel sets × edge_weights.

    Grid size per activation:
      - ``var_default``: 2 configs (noew + ew)
      - ``fixed_3``: 1 config (ew only — fixed C=3 with learnable edge weights)

    With default channel sets and top_n=4 → 4×(2+1) = **12** configs.

    Parameters
    ----------
    top_activations:
        Activation values (e.g. ["relu", None, "gelu", "tanh"]) chosen from
        Phase A results.  Pass the raw activation value, not the label.
    activation_readouts:
        Per-activation output readout from the activation×readout phase
        (e.g. ``{"relu": "mean", "gelu": "mean_minus_median"}``).  When
        omitted, every config uses ``base_cfg.output_readout``.
    with_gd:
        Add periodic gradient descent to every config.
    channel_set_names:
        Subset of CHANNEL_SETS keys to use.  Default: all sets (var_default + fixed_3).
    base_cfg:
        Starting point for every grid entry (default: BASELINE_CFG).
    save_root:
        Override the base save directory.
    """
    root = save_root or str(repo_root() / "runs" / "ablation" / "study")
    if with_gd:
        suffix = f"_gd_{gd_profile}" if gd_profile else "_gd"
    else:
        suffix = ""
    ch_names = channel_set_names or list(CHANNEL_SETS.keys())
    gd_kwargs: dict = {}
    if with_gd:
        gd_kwargs = dict(
            periodic_gd_enabled=True,
            periodic_gd_every=gd_every,
            periodic_gd_epochs=gd_epochs,
            periodic_gd_batch_size=gd_batch,
            periodic_gd_max_batches_per_epoch=gd_max_batches,
            periodic_gd_cv_enabled=gd_cv_enabled,
            periodic_gd_cv_folds=gd_cv_folds,
        )

    grid: dict[str, ExperimentConfig] = {}
    for act in top_activations:
        act_label = ACTIVATION_LABELS.get(act, str(act))
        readout = (
            activation_readouts.get(act, base_cfg.output_readout)
            if activation_readouts
            else base_cfg.output_readout
        )
        for ch_name in ch_names:
            ew_values = CHANNEL_SET_EDGE_WEIGHTS.get(ch_name, (False, True))
            for ew in ew_values:
                ew_label = "ew" if ew else "noew"
                key = f"{act_label}_{ew_label}_{ch_name}{suffix}"
                grid[key] = replace(
                    base_cfg,
                    activation=act,
                    output_readout=readout,
                    edge_weights=ew,
                    allow_variable_channels=CHANNEL_SET_VARIABLE[ch_name],
                    allowed_channels=CHANNEL_SETS[ch_name],
                    save_run_path=f"{root}/{key}",
                    **gd_kwargs,
                )
    return grid


def _resolve_gd_kwargs(
    *,
    with_gd: bool,
    gd_every: int,
    gd_epochs: int,
    gd_batch: int,
    gd_max_batches: int | None,
    gd_cv_enabled: bool,
    gd_cv_folds: int,
) -> dict:
    if not with_gd:
        return {}
    return dict(
        periodic_gd_enabled=True,
        periodic_gd_every=gd_every,
        periodic_gd_epochs=gd_epochs,
        periodic_gd_batch_size=gd_batch,
        periodic_gd_max_batches_per_epoch=gd_max_batches,
        periodic_gd_cv_enabled=gd_cv_enabled,
        periodic_gd_cv_folds=gd_cv_folds,
    )


def make_full_conv_grid(
    *,
    base_cfg: ExperimentConfig = BASELINE_CFG,
    activations: list[str | None] | None = None,
    readouts: list[str] | None = None,
    channel_set_names: list[str] | None = None,
    with_gd: bool = False,
    save_root: str | None = None,
    gd_profile: str | None = None,
    gd_every: int = 5,
    gd_epochs: int = 3,
    gd_batch: int = 256,
    gd_max_batches: int | None = None,
    gd_cv_enabled: bool = False,
    gd_cv_folds: int = 5,
) -> dict[str, "ExperimentConfig"]:
    """Full Cartesian conv grid: activations × readouts × channel sets × edge weights.

    Unlike :func:`make_phase_b_grid` (which sweeps a passed-in activation→readout
    mapping), this builds the complete independent grid.  Used by the OVA class
    study Phase B.

    Grid size with defaults: 5 activations × 2 readouts × (var_default noew+ew,
    fixed_3 ew) = 5 × 2 × 3 = **30** configs.

    Config keys: ``{act_label}_{ro_label}_{noew|ew}_{ch_name}[_gd_{profile}]``.
    """
    root = save_root or str(repo_root() / "runs" / "ablation" / "study")
    acts = activations if activations is not None else ALL_ACTIVATIONS
    ros = readouts if readouts is not None else ALL_READOUTS
    ch_names = channel_set_names or list(CHANNEL_SETS.keys())
    if with_gd:
        suffix = f"_gd_{gd_profile}" if gd_profile else "_gd"
    else:
        suffix = ""
    gd_kwargs = _resolve_gd_kwargs(
        with_gd=with_gd, gd_every=gd_every, gd_epochs=gd_epochs, gd_batch=gd_batch,
        gd_max_batches=gd_max_batches, gd_cv_enabled=gd_cv_enabled, gd_cv_folds=gd_cv_folds,
    )

    grid: dict[str, ExperimentConfig] = {}
    for act in acts:
        act_label = ACTIVATION_LABELS.get(act, str(act))
        for readout in ros:
            ro_label = READOUT_LABELS.get(readout, readout)
            for ch_name in ch_names:
                for ew in CHANNEL_SET_EDGE_WEIGHTS.get(ch_name, (False, True)):
                    ew_label = "ew" if ew else "noew"
                    key = f"{act_label}_{ro_label}_{ew_label}_{ch_name}{suffix}"
                    grid[key] = replace(
                        base_cfg,
                        activation=act,
                        output_readout=readout,
                        edge_weights=ew,
                        allow_variable_channels=CHANNEL_SET_VARIABLE[ch_name],
                        allowed_channels=CHANNEL_SETS[ch_name],
                        save_run_path=f"{root}/{key}",
                        **gd_kwargs,
                    )
    return grid


def make_conv_variants_grid(
    base_configs: list[tuple[str | None, str]],
    *,
    base_cfg: ExperimentConfig = BASELINE_CFG,
    channel_set_names: list[str] | None = None,
    with_gd: bool = False,
    save_root: str | None = None,
    gd_profile: str | None = None,
    gd_every: int = 5,
    gd_epochs: int = 3,
    gd_batch: int = 256,
    gd_max_batches: int | None = None,
    gd_cv_enabled: bool = False,
    gd_cv_folds: int = 5,
) -> dict[str, "ExperimentConfig"]:
    """Expand explicit ``(activation, readout)`` pairs with channel/edge-weight variants.

    Used by the OVA class study Phase D, fed with Phase C's top-5 pilot winners
    so that channels and edge weights are only explored on those best 5 (not all
    activations).  Handles the same activation appearing with different readouts.

    Grid size with 5 pairs and default channel sets: 5 × 3 = **15** configs.

    Config keys: ``{act_label}_{ro_label}_{noew|ew}_{ch_name}[_gd_{profile}]``.
    """
    root = save_root or str(repo_root() / "runs" / "ablation" / "study")
    ch_names = channel_set_names or list(CHANNEL_SETS.keys())
    if with_gd:
        suffix = f"_gd_{gd_profile}" if gd_profile else "_gd"
    else:
        suffix = ""
    gd_kwargs = _resolve_gd_kwargs(
        with_gd=with_gd, gd_every=gd_every, gd_epochs=gd_epochs, gd_batch=gd_batch,
        gd_max_batches=gd_max_batches, gd_cv_enabled=gd_cv_enabled, gd_cv_folds=gd_cv_folds,
    )

    grid: dict[str, ExperimentConfig] = {}
    for act, readout in base_configs:
        act_label = ACTIVATION_LABELS.get(act, str(act))
        ro_label = READOUT_LABELS.get(readout, readout)
        for ch_name in ch_names:
            for ew in CHANNEL_SET_EDGE_WEIGHTS.get(ch_name, (False, True)):
                ew_label = "ew" if ew else "noew"
                key = f"{act_label}_{ro_label}_{ew_label}_{ch_name}{suffix}"
                grid[key] = replace(
                    base_cfg,
                    activation=act,
                    output_readout=readout,
                    edge_weights=ew,
                    allow_variable_channels=CHANNEL_SET_VARIABLE[ch_name],
                    allowed_channels=CHANNEL_SETS[ch_name],
                    save_run_path=f"{root}/{key}",
                    **gd_kwargs,
                )
    return grid