Skip to content

data_model

Data configuration.

Float = Annotated[float, PlainSerializer(np_float_to_scientific_str, return_type=str)] module-attribute #

Annotated float type, used to serialize floats to strings.

DataConfig #

Bases: BaseModel

Data configuration.

If std is specified, mean must be specified as well. Note that setting the std first and then the mean (if they were both None before) will raise a validation error. Prefer instead set_mean_and_std to set both at once. Means and stds are expected to be lists of floats, one for each channel. For supervised tasks, the mean and std of the target could be different from the input data.

All supported transforms are defined in the SupportedTransform enum.

Examples:

Minimum example:

>>> data = DataConfig(
...     data_type="array", # defined in SupportedData
...     patch_size=[128, 128],
...     batch_size=4,
...     axes="YX"
... )

To change the image_means and image_stds of the data:

>>> data.set_means_and_stds(image_means=[214.3], image_stds=[84.5])

One can pass also a list of transformations, by keyword, using the SupportedTransform value:

>>> from careamics.config.support import SupportedTransform
>>> data = DataConfig(
...     data_type="tiff",
...     patch_size=[128, 128],
...     batch_size=4,
...     axes="YX",
...     transforms=[
...         {
...             "name": "XYFlip",
...         }
...     ]
... )
Source code in src/careamics/config/data/data_model.py
class DataConfig(BaseModel):
    """Data configuration.

    If std is specified, mean must be specified as well. Note that setting the std first
    and then the mean (if they were both `None` before) will raise a validation error.
    Prefer instead `set_mean_and_std` to set both at once. Means and stds are expected
    to be lists of floats, one for each channel. For supervised tasks, the mean and std
    of the target could be different from the input data.

    All supported transforms are defined in the SupportedTransform enum.

    Examples
    --------
    Minimum example:

    >>> data = DataConfig(
    ...     data_type="array", # defined in SupportedData
    ...     patch_size=[128, 128],
    ...     batch_size=4,
    ...     axes="YX"
    ... )

    To change the image_means and image_stds of the data:
    >>> data.set_means_and_stds(image_means=[214.3], image_stds=[84.5])

    One can pass also a list of transformations, by keyword, using the
    SupportedTransform value:
    >>> from careamics.config.support import SupportedTransform
    >>> data = DataConfig(
    ...     data_type="tiff",
    ...     patch_size=[128, 128],
    ...     batch_size=4,
    ...     axes="YX",
    ...     transforms=[
    ...         {
    ...             "name": "XYFlip",
    ...         }
    ...     ]
    ... )
    """

    # Pydantic class configuration
    model_config = ConfigDict(
        validate_assignment=True,
    )

    # Dataset configuration
    data_type: Literal["array", "tiff", "custom"]
    """Type of input data, numpy.ndarray (array) or paths (tiff and custom), as defined
    in SupportedData."""

    axes: str
    """Axes of the data, as defined in SupportedAxes."""

    patch_size: Union[list[int]] = Field(..., min_length=2, max_length=3)
    """Patch size, as used during training."""

    batch_size: int = Field(default=1, ge=1, validate_default=True)
    """Batch size for training."""

    # Optional fields
    image_means: Optional[list[Float]] = Field(
        default=None, min_length=0, max_length=32
    )
    """Means of the data across channels, used for normalization."""

    image_stds: Optional[list[Float]] = Field(default=None, min_length=0, max_length=32)
    """Standard deviations of the data across channels, used for normalization."""

    target_means: Optional[list[Float]] = Field(
        default=None, min_length=0, max_length=32
    )
    """Means of the target data across channels, used for normalization."""

    target_stds: Optional[list[Float]] = Field(
        default=None, min_length=0, max_length=32
    )
    """Standard deviations of the target data across channels, used for
    normalization."""

    transforms: Sequence[Union[XYFlipModel, XYRandomRotate90Model]] = Field(
        default=[
            XYFlipModel(),
            XYRandomRotate90Model(),
        ],
        validate_default=True,
    )
    """List of transformations to apply to the data, available transforms are defined
    in SupportedTransform."""

    train_dataloader_params: dict[str, Any] = Field(
        default={"shuffle": True}, validate_default=True
    )
    """Dictionary of PyTorch training dataloader parameters. The dataloader parameters,
    should include the `shuffle` key, which is set to `True` by default. We strongly
    recommend to keep it as `True` to ensure the best training results."""

    val_dataloader_params: dict[str, Any] = Field(default={})
    """Dictionary of PyTorch validation dataloader parameters."""

    @field_validator("patch_size")
    @classmethod
    def all_elements_power_of_2_minimum_8(
        cls, patch_list: Union[list[int]]
    ) -> Union[list[int]]:
        """
        Validate patch size.

        Patch size must be powers of 2 and minimum 8.

        Parameters
        ----------
        patch_list : list of int
            Patch size.

        Returns
        -------
        list of int
            Validated patch size.

        Raises
        ------
        ValueError
            If the patch size is smaller than 8.
        ValueError
            If the patch size is not a power of 2.
        """
        patch_size_ge_than_8_power_of_2(patch_list)

        return patch_list

    @field_validator("axes")
    @classmethod
    def axes_valid(cls, axes: str) -> str:
        """
        Validate axes.

        Axes must:
        - be a combination of 'STCZYX'
        - not contain duplicates
        - contain at least 2 contiguous axes: X and Y
        - contain at most 4 axes
        - not contain both S and T axes

        Parameters
        ----------
        axes : str
            Axes to validate.

        Returns
        -------
        str
            Validated axes.

        Raises
        ------
        ValueError
            If axes are not valid.
        """
        # Validate axes
        check_axes_validity(axes)

        return axes

    @field_validator("train_dataloader_params")
    @classmethod
    def shuffle_train_dataloader(
        cls, train_dataloader_params: dict[str, Any]
    ) -> dict[str, Any]:
        """
        Validate that "shuffle" is included in the training dataloader params.

        A warning will be raised if `shuffle=False`.

        Parameters
        ----------
        train_dataloader_params : dict of {str: Any}
            The training dataloader parameters.

        Returns
        -------
        dict of {str: Any}
            The validated training dataloader parameters.

        Raises
        ------
        ValueError
            If "shuffle" is not included in the training dataloader params.
        """
        if "shuffle" not in train_dataloader_params:
            raise ValueError(
                "Value for 'shuffle' was not included in the `train_dataloader_params`."
            )
        elif ("shuffle" in train_dataloader_params) and (
            not train_dataloader_params["shuffle"]
        ):
            warn(
                "Dataloader parameters include `shuffle=False`, this will be passed to "
                "the training dataloader and may lead to lower quality results.",
                stacklevel=1,
            )
        return train_dataloader_params

    @model_validator(mode="after")
    def std_only_with_mean(self: Self) -> Self:
        """
        Check that mean and std are either both None, or both specified.

        Returns
        -------
        Self
            Validated data model.

        Raises
        ------
        ValueError
            If std is not None and mean is None.
        """
        # check that mean and std are either both None, or both specified
        if (self.image_means and not self.image_stds) or (
            self.image_stds and not self.image_means
        ):
            raise ValueError(
                "Mean and std must be either both None, or both specified."
            )

        elif (self.image_means is not None and self.image_stds is not None) and (
            len(self.image_means) != len(self.image_stds)
        ):
            raise ValueError("Mean and std must be specified for each input channel.")

        if (self.target_means and not self.target_stds) or (
            self.target_stds and not self.target_means
        ):
            raise ValueError(
                "Mean and std must be either both None, or both specified "
            )

        elif self.target_means is not None and self.target_stds is not None:
            if len(self.target_means) != len(self.target_stds):
                raise ValueError(
                    "Mean and std must be either both None, or both specified for each "
                    "target channel."
                )

        return self

    @model_validator(mode="after")
    def validate_dimensions(self: Self) -> Self:
        """
        Validate 2D/3D dimensions between axes, patch size and transforms.

        Returns
        -------
        Self
            Validated data model.

        Raises
        ------
        ValueError
            If the transforms are not valid.
        """
        if "Z" in self.axes:
            if len(self.patch_size) != 3:
                raise ValueError(
                    f"Patch size must have 3 dimensions if the data is 3D "
                    f"({self.axes})."
                )

        else:
            if len(self.patch_size) != 2:
                raise ValueError(
                    f"Patch size must have 3 dimensions if the data is 3D "
                    f"({self.axes})."
                )

        return self

    def __str__(self) -> str:
        """
        Pretty string reprensenting the configuration.

        Returns
        -------
        str
            Pretty string.
        """
        return pformat(self.model_dump())

    def _update(self, **kwargs: Any) -> None:
        """
        Update multiple arguments at once.

        Parameters
        ----------
        **kwargs : Any
            Keyword arguments to update.
        """
        self.__dict__.update(kwargs)
        self.__class__.model_validate(self.__dict__)

    def set_means_and_stds(
        self,
        image_means: Union[NDArray, tuple, list, None],
        image_stds: Union[NDArray, tuple, list, None],
        target_means: Optional[Union[NDArray, tuple, list, None]] = None,
        target_stds: Optional[Union[NDArray, tuple, list, None]] = None,
    ) -> None:
        """
        Set mean and standard deviation of the data across channels.

        This method should be used instead setting the fields directly, as it would
        otherwise trigger a validation error.

        Parameters
        ----------
        image_means : numpy.ndarray, tuple or list
            Mean values for normalization.
        image_stds : numpy.ndarray, tuple or list
            Standard deviation values for normalization.
        target_means : numpy.ndarray, tuple or list, optional
            Target mean values for normalization, by default ().
        target_stds : numpy.ndarray, tuple or list, optional
            Target standard deviation values for normalization, by default ().
        """
        # make sure we pass a list
        if image_means is not None:
            image_means = list(image_means)
        if image_stds is not None:
            image_stds = list(image_stds)
        if target_means is not None:
            target_means = list(target_means)
        if target_stds is not None:
            target_stds = list(target_stds)

        self._update(
            image_means=image_means,
            image_stds=image_stds,
            target_means=target_means,
            target_stds=target_stds,
        )

    def set_3D(self, axes: str, patch_size: list[int]) -> None:
        """
        Set 3D parameters.

        Parameters
        ----------
        axes : str
            Axes.
        patch_size : list of int
            Patch size.
        """
        self._update(axes=axes, patch_size=patch_size)

axes instance-attribute #

Axes of the data, as defined in SupportedAxes.

batch_size = Field(default=1, ge=1, validate_default=True) class-attribute instance-attribute #

Batch size for training.

data_type instance-attribute #

Type of input data, numpy.ndarray (array) or paths (tiff and custom), as defined in SupportedData.

image_means = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Means of the data across channels, used for normalization.

image_stds = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Standard deviations of the data across channels, used for normalization.

patch_size = Field(..., min_length=2, max_length=3) class-attribute instance-attribute #

Patch size, as used during training.

target_means = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Means of the target data across channels, used for normalization.

target_stds = Field(default=None, min_length=0, max_length=32) class-attribute instance-attribute #

Standard deviations of the target data across channels, used for normalization.

train_dataloader_params = Field(default={'shuffle': True}, validate_default=True) class-attribute instance-attribute #

Dictionary of PyTorch training dataloader parameters. The dataloader parameters, should include the shuffle key, which is set to True by default. We strongly recommend to keep it as True to ensure the best training results.

transforms = Field(default=[XYFlipModel(), XYRandomRotate90Model()], validate_default=True) class-attribute instance-attribute #

List of transformations to apply to the data, available transforms are defined in SupportedTransform.

val_dataloader_params = Field(default={}) class-attribute instance-attribute #

Dictionary of PyTorch validation dataloader parameters.

__str__() #

Pretty string reprensenting the configuration.

Returns:

Type Description
str

Pretty string.

Source code in src/careamics/config/data/data_model.py
def __str__(self) -> str:
    """
    Pretty string reprensenting the configuration.

    Returns
    -------
    str
        Pretty string.
    """
    return pformat(self.model_dump())

all_elements_power_of_2_minimum_8(patch_list) classmethod #

Validate patch size.

Patch size must be powers of 2 and minimum 8.

Parameters:

Name Type Description Default
patch_list list of int

Patch size.

required

Returns:

Type Description
list of int

Validated patch size.

Raises:

Type Description
ValueError

If the patch size is smaller than 8.

ValueError

If the patch size is not a power of 2.

Source code in src/careamics/config/data/data_model.py
@field_validator("patch_size")
@classmethod
def all_elements_power_of_2_minimum_8(
    cls, patch_list: Union[list[int]]
) -> Union[list[int]]:
    """
    Validate patch size.

    Patch size must be powers of 2 and minimum 8.

    Parameters
    ----------
    patch_list : list of int
        Patch size.

    Returns
    -------
    list of int
        Validated patch size.

    Raises
    ------
    ValueError
        If the patch size is smaller than 8.
    ValueError
        If the patch size is not a power of 2.
    """
    patch_size_ge_than_8_power_of_2(patch_list)

    return patch_list

axes_valid(axes) classmethod #

Validate axes.

Axes must: - be a combination of 'STCZYX' - not contain duplicates - contain at least 2 contiguous axes: X and Y - contain at most 4 axes - not contain both S and T axes

Parameters:

Name Type Description Default
axes str

Axes to validate.

required

Returns:

Type Description
str

Validated axes.

Raises:

Type Description
ValueError

If axes are not valid.

Source code in src/careamics/config/data/data_model.py
@field_validator("axes")
@classmethod
def axes_valid(cls, axes: str) -> str:
    """
    Validate axes.

    Axes must:
    - be a combination of 'STCZYX'
    - not contain duplicates
    - contain at least 2 contiguous axes: X and Y
    - contain at most 4 axes
    - not contain both S and T axes

    Parameters
    ----------
    axes : str
        Axes to validate.

    Returns
    -------
    str
        Validated axes.

    Raises
    ------
    ValueError
        If axes are not valid.
    """
    # Validate axes
    check_axes_validity(axes)

    return axes

set_3D(axes, patch_size) #

Set 3D parameters.

Parameters:

Name Type Description Default
axes str

Axes.

required
patch_size list of int

Patch size.

required
Source code in src/careamics/config/data/data_model.py
def set_3D(self, axes: str, patch_size: list[int]) -> None:
    """
    Set 3D parameters.

    Parameters
    ----------
    axes : str
        Axes.
    patch_size : list of int
        Patch size.
    """
    self._update(axes=axes, patch_size=patch_size)

set_means_and_stds(image_means, image_stds, target_means=None, target_stds=None) #

Set mean and standard deviation of the data across channels.

This method should be used instead setting the fields directly, as it would otherwise trigger a validation error.

Parameters:

Name Type Description Default
image_means (ndarray, tuple or list)

Mean values for normalization.

required
image_stds (ndarray, tuple or list)

Standard deviation values for normalization.

required
target_means (ndarray, tuple or list)

Target mean values for normalization, by default ().

None
target_stds (ndarray, tuple or list)

Target standard deviation values for normalization, by default ().

None
Source code in src/careamics/config/data/data_model.py
def set_means_and_stds(
    self,
    image_means: Union[NDArray, tuple, list, None],
    image_stds: Union[NDArray, tuple, list, None],
    target_means: Optional[Union[NDArray, tuple, list, None]] = None,
    target_stds: Optional[Union[NDArray, tuple, list, None]] = None,
) -> None:
    """
    Set mean and standard deviation of the data across channels.

    This method should be used instead setting the fields directly, as it would
    otherwise trigger a validation error.

    Parameters
    ----------
    image_means : numpy.ndarray, tuple or list
        Mean values for normalization.
    image_stds : numpy.ndarray, tuple or list
        Standard deviation values for normalization.
    target_means : numpy.ndarray, tuple or list, optional
        Target mean values for normalization, by default ().
    target_stds : numpy.ndarray, tuple or list, optional
        Target standard deviation values for normalization, by default ().
    """
    # make sure we pass a list
    if image_means is not None:
        image_means = list(image_means)
    if image_stds is not None:
        image_stds = list(image_stds)
    if target_means is not None:
        target_means = list(target_means)
    if target_stds is not None:
        target_stds = list(target_stds)

    self._update(
        image_means=image_means,
        image_stds=image_stds,
        target_means=target_means,
        target_stds=target_stds,
    )

shuffle_train_dataloader(train_dataloader_params) classmethod #

Validate that "shuffle" is included in the training dataloader params.

A warning will be raised if shuffle=False.

Parameters:

Name Type Description Default
train_dataloader_params dict of {str: Any}

The training dataloader parameters.

required

Returns:

Type Description
dict of {str: Any}

The validated training dataloader parameters.

Raises:

Type Description
ValueError

If "shuffle" is not included in the training dataloader params.

Source code in src/careamics/config/data/data_model.py
@field_validator("train_dataloader_params")
@classmethod
def shuffle_train_dataloader(
    cls, train_dataloader_params: dict[str, Any]
) -> dict[str, Any]:
    """
    Validate that "shuffle" is included in the training dataloader params.

    A warning will be raised if `shuffle=False`.

    Parameters
    ----------
    train_dataloader_params : dict of {str: Any}
        The training dataloader parameters.

    Returns
    -------
    dict of {str: Any}
        The validated training dataloader parameters.

    Raises
    ------
    ValueError
        If "shuffle" is not included in the training dataloader params.
    """
    if "shuffle" not in train_dataloader_params:
        raise ValueError(
            "Value for 'shuffle' was not included in the `train_dataloader_params`."
        )
    elif ("shuffle" in train_dataloader_params) and (
        not train_dataloader_params["shuffle"]
    ):
        warn(
            "Dataloader parameters include `shuffle=False`, this will be passed to "
            "the training dataloader and may lead to lower quality results.",
            stacklevel=1,
        )
    return train_dataloader_params

std_only_with_mean() #

Check that mean and std are either both None, or both specified.

Returns:

Type Description
Self

Validated data model.

Raises:

Type Description
ValueError

If std is not None and mean is None.

Source code in src/careamics/config/data/data_model.py
@model_validator(mode="after")
def std_only_with_mean(self: Self) -> Self:
    """
    Check that mean and std are either both None, or both specified.

    Returns
    -------
    Self
        Validated data model.

    Raises
    ------
    ValueError
        If std is not None and mean is None.
    """
    # check that mean and std are either both None, or both specified
    if (self.image_means and not self.image_stds) or (
        self.image_stds and not self.image_means
    ):
        raise ValueError(
            "Mean and std must be either both None, or both specified."
        )

    elif (self.image_means is not None and self.image_stds is not None) and (
        len(self.image_means) != len(self.image_stds)
    ):
        raise ValueError("Mean and std must be specified for each input channel.")

    if (self.target_means and not self.target_stds) or (
        self.target_stds and not self.target_means
    ):
        raise ValueError(
            "Mean and std must be either both None, or both specified "
        )

    elif self.target_means is not None and self.target_stds is not None:
        if len(self.target_means) != len(self.target_stds):
            raise ValueError(
                "Mean and std must be either both None, or both specified for each "
                "target channel."
            )

    return self

validate_dimensions() #

Validate 2D/3D dimensions between axes, patch size and transforms.

Returns:

Type Description
Self

Validated data model.

Raises:

Type Description
ValueError

If the transforms are not valid.

Source code in src/careamics/config/data/data_model.py
@model_validator(mode="after")
def validate_dimensions(self: Self) -> Self:
    """
    Validate 2D/3D dimensions between axes, patch size and transforms.

    Returns
    -------
    Self
        Validated data model.

    Raises
    ------
    ValueError
        If the transforms are not valid.
    """
    if "Z" in self.axes:
        if len(self.patch_size) != 3:
            raise ValueError(
                f"Patch size must have 3 dimensions if the data is 3D "
                f"({self.axes})."
            )

    else:
        if len(self.patch_size) != 2:
            raise ValueError(
                f"Patch size must have 3 dimensions if the data is 3D "
                f"({self.axes})."
            )

    return self

np_float_to_scientific_str(x) #

Return a string scientific representation of a float.

In particular, this method is used to serialize floats to strings, allowing numpy.float32 to be passed in the Pydantic model and written to a yaml file as str.

Parameters:

Name Type Description Default
x float

Input value.

required

Returns:

Type Description
str

Scientific string representation of the input value.

Source code in src/careamics/config/data/data_model.py
def np_float_to_scientific_str(x: float) -> str:
    """Return a string scientific representation of a float.

    In particular, this method is used to serialize floats to strings, allowing
    numpy.float32 to be passed in the Pydantic model and written to a yaml file as str.

    Parameters
    ----------
    x : float
        Input value.

    Returns
    -------
    str
        Scientific string representation of the input value.
    """
    return np.format_float_scientific(x, precision=7)