Skip to content

ng_data_model

Data configuration.

Float = Annotated[float, PlainSerializer(np_float_to_scientific_str, return_type=str)] module-attribute #

Annotated float type, used to serialize floats to strings.

PatchingStrategies = Union[RandomPatchingModel, TiledPatchingModel, WholePatchingModel] module-attribute #

Patching strategies.

NGDataConfig #

Bases: BaseModel

Next-Generation Dataset configuration.

NGDataConfig are used for both training and prediction, with the patching strategy determining how the data is processed. Note that random is the only patching strategy compatible with training, while tiled and whole are only used for prediction.

If std is specified, mean must be specified as well. Note that setting the std first and then the mean (if they were both None before) will raise a validation error. Prefer instead set_means_and_stds to set both at once. Means and stds are expected to be lists of floats, one for each channel. For supervised tasks, the mean and std of the target could be different from the input data.

All supported transforms are defined in the SupportedTransform enum.

Source code in src/careamics/config/data/ng_data_model.py
class NGDataConfig(BaseModel):
    """Next-Generation Dataset configuration.

    NGDataConfig are used for both training and prediction, with the patching strategy
    determining how the data is processed. Note that `random` is the only patching
    strategy compatible with training, while `tiled` and `whole` are only used for
    prediction.

    If std is specified, mean must be specified as well. Note that setting the std first
    and then the mean (if they were both `None` before) will raise a validation error.
    Prefer instead `set_means_and_stds` to set both at once. Means and stds are expected
    to be lists of floats, one for each channel. For supervised tasks, the mean and std
    of the target could be different from the input data.

    All supported transforms are defined in the SupportedTransform enum.
    """

    # Pydantic class configuration
    model_config = ConfigDict(
        validate_assignment=True,
    )

    # Dataset configuration
    data_type: Literal["array", "tiff", "zarr", "custom"]
    """Type of input data."""

    axes: str
    """Axes of the data, as defined in SupportedAxes."""

    patching: PatchingStrategies = Field(..., discriminator="name")
    """Patching strategy to use. Note that `random` is the only supported strategy for
    training, while `tiled` and `whole` are only used for prediction."""

    # Optional fields
    batch_size: int = Field(default=1, ge=1, validate_default=True)
    """Batch size for training."""

    image_means: Optional[list[Float]] = Field(
        default=None, min_length=0, max_length=32
    )
    """Means of the data across channels, used for normalization."""

    image_stds: Optional[list[Float]] = Field(default=None, min_length=0, max_length=32)
    """Standard deviations of the data across channels, used for normalization."""

    target_means: Optional[list[Float]] = Field(
        default=None, min_length=0, max_length=32
    )
    """Means of the target data across channels, used for normalization."""

    target_stds: Optional[list[Float]] = Field(
        default=None, min_length=0, max_length=32
    )
    """Standard deviations of the target data across channels, used for
    normalization."""

    transforms: Sequence[Union[XYFlipModel, XYRandomRotate90Model]] = Field(
        default=(
            XYFlipModel(),
            XYRandomRotate90Model(),
        ),
        validate_default=True,
    )
    """List of transformations to apply to the data, available transforms are defined
    in SupportedTransform."""

    train_dataloader_params: dict[str, Any] = Field(
        default={"shuffle": True}, validate_default=True
    )
    """Dictionary of PyTorch training dataloader parameters. The dataloader parameters,
    should include the `shuffle` key, which is set to `True` by default. We strongly
    recommend to keep it as `True` to ensure the best training results."""

    val_dataloader_params: dict[str, Any] = Field(default={})
    """Dictionary of PyTorch validation dataloader parameters."""

    test_dataloader_params: dict[str, Any] = Field(default={})
    """Dictionary of PyTorch test dataloader parameters."""

    seed: Optional[int] = Field(default=None, gt=0)
    """Random seed for reproducibility."""

    @field_validator("axes")
    @classmethod
    def axes_valid(cls, axes: str) -> str:
        """
        Validate axes.

        Axes must:
        - be a combination of 'STCZYX'
        - not contain duplicates
        - contain at least 2 contiguous axes: X and Y
        - contain at most 4 axes
        - not contain both S and T axes

        Parameters
        ----------
        axes : str
            Axes to validate.

        Returns
        -------
        str
            Validated axes.

        Raises
        ------
        ValueError
            If axes are not valid.
        """
        # Validate axes
        check_axes_validity(axes)

        return axes

    @field_validator("train_dataloader_params")
    @classmethod
    def shuffle_train_dataloader(
        cls, train_dataloader_params: dict[str, Any]
    ) -> dict[str, Any]:
        """
        Validate that "shuffle" is included in the training dataloader params.

        A warning will be raised if `shuffle=False`.

        Parameters
        ----------
        train_dataloader_params : dict of {str: Any}
            The training dataloader parameters.

        Returns
        -------
        dict of {str: Any}
            The validated training dataloader parameters.

        Raises
        ------
        ValueError
            If "shuffle" is not included in the training dataloader params.
        """
        if "shuffle" not in train_dataloader_params:
            raise ValueError(
                "Value for 'shuffle' was not included in the `train_dataloader_params`."
            )
        elif ("shuffle" in train_dataloader_params) and (
            not train_dataloader_params["shuffle"]
        ):
            warn(
                "Dataloader parameters include `shuffle=False`, this will be passed to "
                "the training dataloader and may lead to lower quality results.",
                stacklevel=1,
            )
        return train_dataloader_params

    @model_validator(mode="after")
    def std_only_with_mean(self: Self) -> Self:
        """
        Check that mean and std are either both None, or both specified.

        Returns
        -------
        Self
            Validated data model.

        Raises
        ------
        ValueError
            If std is not None and mean is None.
        """
        # check that mean and std are either both None, or both specified
        if (self.image_means and not self.image_stds) or (
            self.image_stds and not self.image_means
        ):
            raise ValueError(
                "Mean and std must be either both None, or both specified."
            )

        elif (self.image_means is not None and self.image_stds is not None) and (
            len(self.image_means) != len(self.image_stds)
        ):
            raise ValueError("Mean and std must be specified for each input channel.")

        if (self.target_means and not self.target_stds) or (
            self.target_stds and not self.target_means
        ):
            raise ValueError(
                "Mean and std must be either both None, or both specified "
            )

        elif self.target_means is not None and self.target_stds is not None:
            if len(self.target_means) != len(self.target_stds):
                raise ValueError(
                    "Mean and std must be either both None, or both specified for each "
                    "target channel."
                )

        return self

    @model_validator(mode="after")
    def validate_dimensions(self: Self) -> Self:
        """
        Validate 2D/3D dimensions between axes and patch size.

        Returns
        -------
        Self
            Validated data model.

        Raises
        ------
        ValueError
            If the patch size dimension is not compatible with the axes.
        """
        if "Z" in self.axes:
            if (
                hasattr(self.patching, "patch_size")
                and len(self.patching.patch_size) != 3
            ):
                raise ValueError(
                    f"`patch_size` in `patching` must have 3 dimensions if the data is"
                    f" 3D, got axes {self.axes})."
                )
        else:
            if (
                hasattr(self.patching, "patch_size")
                and len(self.patching.patch_size) != 2
            ):
                raise ValueError(
                    f"`patch_size` in `patching` must have 2 dimensions if the data is"
                    f" 3D, got axes {self.axes})."
                )

        return self

    def __str__(self) -> str:
        """
        Pretty string reprensenting the configuration.

        Returns
        -------
        str
            Pretty string.
        """
        return pformat(self.model_dump())

    def _update(self, **kwargs: Any) -> None:
        """
        Update multiple arguments at once.

        Parameters
        ----------
        **kwargs : Any
            Keyword arguments to update.
        """
        self.__dict__.update(kwargs)
        self.__class__.model_validate(self.__dict__)

    def set_means_and_stds(
        self,
        image_means: Union[NDArray, tuple, list, None],
        image_stds: Union[NDArray, tuple, list, None],
        target_means: Optional[Union[NDArray, tuple, list, None]] = None,
        target_stds: Optional[Union[NDArray, tuple, list, None]] = None,
    ) -> None:
        """
        Set mean and standard deviation of the data across channels.

        This method should be used instead setting the fields directly, as it would
        otherwise trigger a validation error.

        Parameters
        ----------
        image_means : numpy.ndarray, tuple or list
            Mean values for normalization.
        image_stds : numpy.ndarray, tuple or list
            Standard deviation values for normalization.
        target_means : numpy.ndarray, tuple or list, optional
            Target mean values for normalization, by default ().
        target_stds : numpy.ndarray, tuple or list, optional
            Target standard deviation values for normalization, by default ().
        """
        # make sure we pass a list
        if image_means is not None:
            image_means = list(image_means)
        if image_stds is not None:
            image_stds = list(image_stds)
        if target_means is not None:
            target_means = list(target_means)
        if target_stds is not None:
            target_stds = list(target_stds)

        self._update(
            image_means=image_means,
            image_stds=image_stds,
            target_means=target_means,
            target_stds=target_stds,
        )

axes instance-attribute #

Axes of the data, as defined in SupportedAxes.

batch_size = Field(default=1, ge=1, validate_default=True) class-attribute instance-attribute #

Batch size for training.

data_type instance-attribute #

Type of input data.

image_means = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Means of the data across channels, used for normalization.

image_stds = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Standard deviations of the data across channels, used for normalization.

patching = Field(..., discriminator='name') class-attribute instance-attribute #

Patching strategy to use. Note that random is the only supported strategy for training, while tiled and whole are only used for prediction.

seed = Field(default=None, gt=0) class-attribute instance-attribute #

Random seed for reproducibility.

target_means = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Means of the target data across channels, used for normalization.

target_stds = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Standard deviations of the target data across channels, used for normalization.

test_dataloader_params = Field(default={}) class-attribute instance-attribute #

Dictionary of PyTorch test dataloader parameters.

train_dataloader_params = Field(default={'shuffle': True}, validate_default=True) class-attribute instance-attribute #

Dictionary of PyTorch training dataloader parameters. The dataloader parameters, should include the shuffle key, which is set to True by default. We strongly recommend to keep it as True to ensure the best training results.

transforms = Field(default=(XYFlipModel(), XYRandomRotate90Model()), validate_default=True) class-attribute instance-attribute #

List of transformations to apply to the data, available transforms are defined in SupportedTransform.

val_dataloader_params = Field(default={}) class-attribute instance-attribute #

Dictionary of PyTorch validation dataloader parameters.

__str__() #

Pretty string reprensenting the configuration.

Returns:

Type Description
str

Pretty string.

Source code in src/careamics/config/data/ng_data_model.py
def __str__(self) -> str:
    """
    Pretty string reprensenting the configuration.

    Returns
    -------
    str
        Pretty string.
    """
    return pformat(self.model_dump())

axes_valid(axes) classmethod #

Validate axes.

Axes must: - be a combination of 'STCZYX' - not contain duplicates - contain at least 2 contiguous axes: X and Y - contain at most 4 axes - not contain both S and T axes

Parameters:

Name Type Description Default
axes str

Axes to validate.

required

Returns:

Type Description
str

Validated axes.

Raises:

Type Description
ValueError

If axes are not valid.

Source code in src/careamics/config/data/ng_data_model.py
@field_validator("axes")
@classmethod
def axes_valid(cls, axes: str) -> str:
    """
    Validate axes.

    Axes must:
    - be a combination of 'STCZYX'
    - not contain duplicates
    - contain at least 2 contiguous axes: X and Y
    - contain at most 4 axes
    - not contain both S and T axes

    Parameters
    ----------
    axes : str
        Axes to validate.

    Returns
    -------
    str
        Validated axes.

    Raises
    ------
    ValueError
        If axes are not valid.
    """
    # Validate axes
    check_axes_validity(axes)

    return axes

set_means_and_stds(image_means, image_stds, target_means=None, target_stds=None) #

Set mean and standard deviation of the data across channels.

This method should be used instead setting the fields directly, as it would otherwise trigger a validation error.

Parameters:

Name Type Description Default
image_means (ndarray, tuple or list)

Mean values for normalization.

required
image_stds (ndarray, tuple or list)

Standard deviation values for normalization.

required
target_means (ndarray, tuple or list)

Target mean values for normalization, by default ().

None
target_stds (ndarray, tuple or list)

Target standard deviation values for normalization, by default ().

None
Source code in src/careamics/config/data/ng_data_model.py
def set_means_and_stds(
    self,
    image_means: Union[NDArray, tuple, list, None],
    image_stds: Union[NDArray, tuple, list, None],
    target_means: Optional[Union[NDArray, tuple, list, None]] = None,
    target_stds: Optional[Union[NDArray, tuple, list, None]] = None,
) -> None:
    """
    Set mean and standard deviation of the data across channels.

    This method should be used instead setting the fields directly, as it would
    otherwise trigger a validation error.

    Parameters
    ----------
    image_means : numpy.ndarray, tuple or list
        Mean values for normalization.
    image_stds : numpy.ndarray, tuple or list
        Standard deviation values for normalization.
    target_means : numpy.ndarray, tuple or list, optional
        Target mean values for normalization, by default ().
    target_stds : numpy.ndarray, tuple or list, optional
        Target standard deviation values for normalization, by default ().
    """
    # make sure we pass a list
    if image_means is not None:
        image_means = list(image_means)
    if image_stds is not None:
        image_stds = list(image_stds)
    if target_means is not None:
        target_means = list(target_means)
    if target_stds is not None:
        target_stds = list(target_stds)

    self._update(
        image_means=image_means,
        image_stds=image_stds,
        target_means=target_means,
        target_stds=target_stds,
    )

shuffle_train_dataloader(train_dataloader_params) classmethod #

Validate that "shuffle" is included in the training dataloader params.

A warning will be raised if shuffle=False.

Parameters:

Name Type Description Default
train_dataloader_params dict of {str: Any}

The training dataloader parameters.

required

Returns:

Type Description
dict of {str: Any}

The validated training dataloader parameters.

Raises:

Type Description
ValueError

If "shuffle" is not included in the training dataloader params.

Source code in src/careamics/config/data/ng_data_model.py
@field_validator("train_dataloader_params")
@classmethod
def shuffle_train_dataloader(
    cls, train_dataloader_params: dict[str, Any]
) -> dict[str, Any]:
    """
    Validate that "shuffle" is included in the training dataloader params.

    A warning will be raised if `shuffle=False`.

    Parameters
    ----------
    train_dataloader_params : dict of {str: Any}
        The training dataloader parameters.

    Returns
    -------
    dict of {str: Any}
        The validated training dataloader parameters.

    Raises
    ------
    ValueError
        If "shuffle" is not included in the training dataloader params.
    """
    if "shuffle" not in train_dataloader_params:
        raise ValueError(
            "Value for 'shuffle' was not included in the `train_dataloader_params`."
        )
    elif ("shuffle" in train_dataloader_params) and (
        not train_dataloader_params["shuffle"]
    ):
        warn(
            "Dataloader parameters include `shuffle=False`, this will be passed to "
            "the training dataloader and may lead to lower quality results.",
            stacklevel=1,
        )
    return train_dataloader_params

std_only_with_mean() #

Check that mean and std are either both None, or both specified.

Returns:

Type Description
Self

Validated data model.

Raises:

Type Description
ValueError

If std is not None and mean is None.

Source code in src/careamics/config/data/ng_data_model.py
@model_validator(mode="after")
def std_only_with_mean(self: Self) -> Self:
    """
    Check that mean and std are either both None, or both specified.

    Returns
    -------
    Self
        Validated data model.

    Raises
    ------
    ValueError
        If std is not None and mean is None.
    """
    # check that mean and std are either both None, or both specified
    if (self.image_means and not self.image_stds) or (
        self.image_stds and not self.image_means
    ):
        raise ValueError(
            "Mean and std must be either both None, or both specified."
        )

    elif (self.image_means is not None and self.image_stds is not None) and (
        len(self.image_means) != len(self.image_stds)
    ):
        raise ValueError("Mean and std must be specified for each input channel.")

    if (self.target_means and not self.target_stds) or (
        self.target_stds and not self.target_means
    ):
        raise ValueError(
            "Mean and std must be either both None, or both specified "
        )

    elif self.target_means is not None and self.target_stds is not None:
        if len(self.target_means) != len(self.target_stds):
            raise ValueError(
                "Mean and std must be either both None, or both specified for each "
                "target channel."
            )

    return self

validate_dimensions() #

Validate 2D/3D dimensions between axes and patch size.

Returns:

Type Description
Self

Validated data model.

Raises:

Type Description
ValueError

If the patch size dimension is not compatible with the axes.

Source code in src/careamics/config/data/ng_data_model.py
@model_validator(mode="after")
def validate_dimensions(self: Self) -> Self:
    """
    Validate 2D/3D dimensions between axes and patch size.

    Returns
    -------
    Self
        Validated data model.

    Raises
    ------
    ValueError
        If the patch size dimension is not compatible with the axes.
    """
    if "Z" in self.axes:
        if (
            hasattr(self.patching, "patch_size")
            and len(self.patching.patch_size) != 3
        ):
            raise ValueError(
                f"`patch_size` in `patching` must have 3 dimensions if the data is"
                f" 3D, got axes {self.axes})."
            )
    else:
        if (
            hasattr(self.patching, "patch_size")
            and len(self.patching.patch_size) != 2
        ):
            raise ValueError(
                f"`patch_size` in `patching` must have 2 dimensions if the data is"
                f" 3D, got axes {self.axes})."
            )

    return self

np_float_to_scientific_str(x) #

Return a string scientific representation of a float.

In particular, this method is used to serialize floats to strings, allowing numpy.float32 to be passed in the Pydantic model and written to a yaml file as str.

Parameters:

Name Type Description Default
x float

Input value.

required

Returns:

Type Description
str

Scientific string representation of the input value.

Source code in src/careamics/config/data/ng_data_model.py
def np_float_to_scientific_str(x: float) -> str:
    """Return a string scientific representation of a float.

    In particular, this method is used to serialize floats to strings, allowing
    numpy.float32 to be passed in the Pydantic model and written to a yaml file as str.

    Parameters
    ----------
    x : float
        Input value.

    Returns
    -------
    str
        Scientific string representation of the input value.
    """
    return np.format_float_scientific(x, precision=7)