Source code for dendritic_modeling.networks.activations.factory

"""
Activation Factory.

This module provides a factory for building different activation/reactivation modules.
"""

import inspect
from collections.abc import Callable, Mapping
from typing import Any

import torch.nn as nn

from dendritic_modeling.networks.activations.endpoint_tanh import EndpointNormalizedTanh
from dendritic_modeling.networks.activations.parametric import (
    ParametricLinearSigmoid,
    ParametricLinearTanh,
    ParametricLinearTanhTransition,
    ParametricReLU,
    ParametricTanh,
    ParametricTanhOnlyM,
    ResidualSiLU,
)
from dendritic_modeling.plugins import load_plugins

ActivationBuilder = Callable[..., nn.Module]


def _require_output_dim(output_dim: int | None, activation_name: str) -> int:
    if output_dim is None:
        raise ValueError(f"{activation_name} requires output_dim.")
    return output_dim


def _normalize_activation_type(act_type: Any) -> str:
    if act_type is None:
        raise ValueError(
            "Activation type must be a string, got None. "
            "Use 'none' for an identity/no-op activation."
        )
    return str(act_type).lower()


def _resolve_activation_builder(act_type: Any) -> ActivationBuilder:
    normalized_type = _normalize_activation_type(act_type)
    return get_activation_builder(normalized_type)


def _identity_builder(**_: Any) -> nn.Module:
    return nn.Identity()


def _relu_builder(**_: Any) -> nn.Module:
    return nn.ReLU()


def _sigmoid_builder(**_: Any) -> nn.Module:
    return nn.Sigmoid()


def _tanh_builder(**_: Any) -> nn.Module:
    return nn.Tanh()


def _gelu_builder(**_: Any) -> nn.Module:
    return nn.GELU()


def _silu_builder(**_: Any) -> nn.Module:
    return nn.SiLU()


def _softplus_builder(**_: Any) -> nn.Module:
    return nn.Softplus()


def _param_tanh_builder(
    *,
    output_dim: int | None,
    init_m: float,
    init_b: float,
    memory_efficient: bool = False,
    **_: Any,
) -> nn.Module:
    return ParametricTanh(
        _require_output_dim(output_dim, "ParametricTanh"),
        init_m,
        init_b,
        memory_efficient=memory_efficient,
    )


def _param_relu_builder(
    *,
    output_dim: int | None,
    init_m: float,
    init_b: float,
    memory_efficient: bool = False,
    **_: Any,
) -> nn.Module:
    return ParametricReLU(
        _require_output_dim(output_dim, "ParametricReLU"),
        init_m,
        init_b,
        memory_efficient=memory_efficient,
    )


def _param_tanh_only_m_builder(
    *,
    output_dim: int | None,
    init_m: float,
    fixed_b: float,
    memory_efficient: bool = False,
    **_: Any,
) -> nn.Module:
    return ParametricTanhOnlyM(
        _require_output_dim(output_dim, "ParametricTanhOnlyM"),
        init_m,
        fixed_b,
        memory_efficient=memory_efficient,
    )


def _param_linear_sigmoid_builder(
    *,
    output_dim: int | None,
    init_m: float,
    init_b: float,
    fixed_b: float,
    memory_efficient: bool = False,
    **_: Any,
) -> nn.Module:
    return ParametricLinearSigmoid(
        _require_output_dim(output_dim, "ParametricLinearSigmoid"),
        init_m=init_m,
        init_b=init_b,
        trainable=True,
    )


def _linear_sigmoid_builder(
    *,
    output_dim: int | None,
    init_m: float,
    init_b: float,
    fixed_b: float,
    memory_efficient: bool = False,
    **_: Any,
) -> nn.Module:
    return ParametricLinearSigmoid(
        _require_output_dim(output_dim, "ParametricLinearSigmoid"),
        init_m=init_m,
        init_b=init_b,
        trainable=False,
    )


def _param_linear_tanh_builder(
    *,
    output_dim: int | None,
    init_m: float,
    init_b: float,
    fixed_b: float,
    memory_efficient: bool = False,
    **_: Any,
) -> nn.Module:
    return ParametricLinearTanh(
        _require_output_dim(output_dim, "ParametricLinearTanh"),
        init_m=init_m,
        init_b=init_b,
        trainable=True,
    )


def _linear_tanh_builder(
    *,
    output_dim: int | None,
    init_m: float,
    init_b: float,
    fixed_b: float,
    memory_efficient: bool = False,
    **_: Any,
) -> nn.Module:
    return ParametricLinearTanh(
        _require_output_dim(output_dim, "ParametricLinearTanh"),
        init_m=init_m,
        init_b=init_b,
        trainable=False,
    )


def _param_linear_tanh_transition_builder(
    *, output_dim: int | None, init_m: float, init_b: float, **_: Any
) -> nn.Module:
    # Map factory parameters to ParametricLinearTanhTransition parameters:
    # - init_b maps to init_threshold (the point where transition happens)
    # - init_m maps to init_transition_slope (controls transition curve steepness)
    return ParametricLinearTanhTransition(
        output_dim,
        init_threshold=init_b,
        linear_slope=1.0,
        init_transition_slope=init_m,
        trainable_threshold=True,
        trainable_transition_slope=True,
    )


def _linear_tanh_transition_builder(
    *, output_dim: int | None, init_m: float, init_b: float, **_: Any
) -> nn.Module:
    # Map factory parameters to ParametricLinearTanhTransition parameters:
    # - init_b maps to init_threshold (the point where transition happens)
    # - init_m maps to init_transition_slope (controls transition curve steepness)
    return ParametricLinearTanhTransition(
        output_dim,
        init_threshold=init_b,
        linear_slope=1.0,
        init_transition_slope=init_m,
        trainable_threshold=False,
        trainable_transition_slope=False,
    )


def _residual_silu_builder(
    *,
    output_dim: int | None,
    init_mix: float = 0.0,
    **_: Any,
) -> nn.Module:
    """Opt-in: identity at construction, SiLU at mix 1 (see ResidualSiLU)."""

    return ResidualSiLU(_require_output_dim(output_dim, "ResidualSiLU"), init_mix)


def _endpoint_tanh_builder(*, output_dim, init_m=3.0, init_b=1.5, **_):
    return EndpointNormalizedTanh(
        _require_output_dim(output_dim, "EndpointNormalizedTanh"), init_m, init_b
    )


_ACTIVATION_REGISTRY: dict[str, ActivationBuilder] = {
    "endpoint_tanh": _endpoint_tanh_builder,
    "gelu": _gelu_builder,
    "linear_sigmoid": _linear_sigmoid_builder,
    "linear_tanh": _linear_tanh_builder,
    "linear_tanh_transition": _linear_tanh_transition_builder,
    "none": _identity_builder,
    "param_linear_sigmoid": _param_linear_sigmoid_builder,
    "param_linear_tanh": _param_linear_tanh_builder,
    "param_linear_tanh_transition": _param_linear_tanh_transition_builder,
    "param_relu": _param_relu_builder,
    "param_tanh": _param_tanh_builder,
    "param_tanh_only_m": _param_tanh_only_m_builder,
    "relu": _relu_builder,
    "residual_silu": _residual_silu_builder,
    "sigmoid": _sigmoid_builder,
    "silu": _silu_builder,
    "softplus": _softplus_builder,
    "swish": _silu_builder,
    "tanh": _tanh_builder,
}


[docs] def register_activation_builder( act_type: str, builder: ActivationBuilder, *, allow_override: bool = False, ) -> None: """Register an activation builder by normalized activation type.""" normalized_type = _normalize_activation_type(act_type) if not callable(builder): raise TypeError(f"Activation builder for {normalized_type!r} must be callable") if normalized_type in _ACTIVATION_REGISTRY and not allow_override: raise ValueError( f"Activation builder '{normalized_type}' is already registered" ) _ACTIVATION_REGISTRY[normalized_type] = builder
def unregister_activation_builder(act_type: str) -> None: """Remove an activation builder if present.""" _ACTIVATION_REGISTRY.pop(_normalize_activation_type(act_type), None) def get_activation_builder(act_type: str) -> ActivationBuilder: """Return a registered activation builder.""" normalized_type = _normalize_activation_type(act_type) builder = _ACTIVATION_REGISTRY.get(normalized_type) if builder is None: load_plugins() builder = _ACTIVATION_REGISTRY.get(normalized_type) if builder is None: raise ValueError(f"Unknown activation type: {normalized_type}") return builder def get_registered_activation_types() -> list[str]: """Return registered activation type names, including those of plugins.""" load_plugins() return sorted(_ACTIVATION_REGISTRY) def unused_activation_options(act_type: Any, options: Mapping[str, Any]) -> list[str]: """Return option names that ``ActivationFactory.create`` would discard. Builders name the options they use; a catch-all ``**_`` parameter discards the rest. Builders that forward ``**kwargs`` to a module constructor keep every option, and that constructor validates them. """ builder = _resolve_activation_builder(act_type) named = {"fixed_b"} forwards_extra = False for name, parameter in inspect.signature(builder).parameters.items(): if parameter.kind is inspect.Parameter.VAR_KEYWORD: forwards_extra = name != "_" elif parameter.kind is not inspect.Parameter.VAR_POSITIONAL: named.add(name) if forwards_extra: return [] return sorted(set(options) - named) class ActivationFactory: """ A factory for building different activation/ reactivation modules. """ @staticmethod def create( act_type, output_dim=None, init_m=1.5, init_b=1.0, fixed_b=0.5, **kwargs ): """ Create an activation module of the desired type. Parameters ---------- act_type : str One of ["none", "param_tanh", "param_tanh_only_m", "relu", "sigmoid", "tanh"]. output_dim : int or None Used by param_tanh and param_tanh_only_m (must be set if act_type is "param_tanh" or "param_tanh_only_m"). init_m : float Initial value for m. init_b : float Initial value for b (used only for param_tanh). fixed_b : float The fixed value of b when using param_tanh_only_m. **kwargs : dict Additional keyword arguments are accepted but ignored (for backwards compatibility). Returns ------- nn.Module The activation module. """ builder = _resolve_activation_builder(act_type) return builder( output_dim=output_dim, init_m=init_m, init_b=init_b, fixed_b=fixed_b, **kwargs, ) __all__ = [ "ActivationFactory", "get_activation_builder", "get_registered_activation_types", "register_activation_builder", "unregister_activation_builder", "unused_activation_options", ]