Source code for dendritic_modeling.datasets.registry

"""Dataset-loader registry for project extensions.

Built-in datasets continue through the established loaders in
``standard_datasets`` and ``sequence_datasets``.  This registry gives new
datasets a config-only entry point without editing those long dispatch chains.
"""

from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass
from typing import Any

from torch.utils.data import Dataset

from dendritic_modeling.plugins import load_plugins

DatasetTriplet = tuple[Dataset, Dataset, Dataset]
DatasetBuilder = Callable[[Any], DatasetTriplet]


@dataclass(frozen=True)
class DatasetSpec:
    """Metadata for a registered dataset loader."""

    name: str
    builder: DatasetBuilder
    description: str = ""
    sequence: bool = False
    # Training strategy the dataset's training split is built for, if any.
    required_strategy: str | None = None
    # data.processing options (flatten, normalize) the builder applies; None
    # leaves them unchecked.
    processing: frozenset[str] | None = None


_DATASET_REGISTRY: dict[str, DatasetSpec] = {}


def _builtin_dataset_names() -> set[str]:
    """Names the built-in loaders handle (imported lazily: they import torch data)."""
    from dendritic_modeling.datasets import standard_datasets
    from dendritic_modeling.datasets.sequence.builders.registry import (
        SEQUENCE_DATASET_BUILDERS,
    )

    return {
        *standard_datasets._STANDARD_VISION_DATASET_LOADERS,
        *standard_datasets._SPECIAL_DATASET_LOADERS,
        *standard_datasets._THEORETICAL_SYNTHETIC_DATASET_NAMES,
        *SEQUENCE_DATASET_BUILDERS,
    }


[docs] def register_dataset( name: str, builder: DatasetBuilder, *, aliases: tuple[str, ...] | list[str] = (), description: str = "", sequence: bool = False, required_strategy: str | None = None, processing: frozenset[str] | set[str] | tuple[str, ...] | None = None, allow_override: bool = False, ) -> None: """Register a dataset builder. Builders receive the original task/data config object and must return ``(train_ds, valid_ds, test_ds)``. """ names = [name, *aliases] if not names or any(not str(item).strip() for item in names): raise ValueError("dataset name and aliases must be non-empty") builtin = _builtin_dataset_names() for raw_name in names: key = str(raw_name).lower() if key in builtin and not allow_override: raise ValueError( f"Dataset '{key}' is a built-in dataset; pass allow_override=True " "to replace its loader" ) if key in _DATASET_REGISTRY and not allow_override: raise ValueError(f"Dataset '{key}' is already registered") _DATASET_REGISTRY[key] = DatasetSpec( name=key, builder=builder, description=description, sequence=sequence, required_strategy=required_strategy, processing=None if processing is None else frozenset(processing), )
def unregister_dataset(name: str) -> None: """Remove a registered dataset if present.""" _DATASET_REGISTRY.pop(str(name).lower(), None) def get_dataset_spec(name: str) -> DatasetSpec | None: """Return a registered dataset spec, if any.""" key = str(name).lower() if key not in _DATASET_REGISTRY: load_plugins() return _DATASET_REGISTRY.get(key) def reject_unsupported_training_strategy(dataset_name: str, strategy: str) -> None: """Raise when a registered dataset requires a different training strategy.""" spec = get_dataset_spec(dataset_name) if spec is None or spec.required_strategy is None: return if str(strategy).strip().lower() != spec.required_strategy: raise ValueError( f"dataset {spec.name!r} must be trained with " f"training.main.strategy: {spec.required_strategy}, got {strategy!r}" ) def get_registered_dataset_names(*, sequence: bool | None = None) -> list[str]: """Return registered dataset names, including those of plugins, optionally filtered by sequence flag.""" load_plugins() if sequence is None: return sorted(_DATASET_REGISTRY) return sorted( name for name, spec in _DATASET_REGISTRY.items() if spec.sequence == sequence ) def build_registered_dataset(name: str, task_cfg: Any) -> DatasetTriplet | None: """Build a registered dataset, or ``None`` if the name is unregistered.""" spec = get_dataset_spec(name) if spec is None: return None return spec.builder(task_cfg) def _build_contextual_remapping(task_cfg: Any) -> DatasetTriplet: # Imported lazily: the module loads torchvision only when the task is used. from .contextual_remapping import build_contextual_remapping_datasets return build_contextual_remapping_datasets(task_cfg) # Its training split serves presentations through ``.stream``; indexing it # returns uncorrupted context-0 items meant only for shape inference. register_dataset( "contextual_remapping", _build_contextual_remapping, description=( "EMNIST Balanced with 32 cyclic context label shifts and per-presentation " "gain/noise corruption" ), required_strategy="presentation_stream", processing=frozenset(), ) __all__ = [ "DatasetBuilder", "DatasetSpec", "DatasetTriplet", "build_registered_dataset", "get_dataset_spec", "get_registered_dataset_names", "register_dataset", "reject_unsupported_training_strategy", "unregister_dataset", ]