[Collectors] Histogram Aggregator

src/nncf/experimental/common/tensor_statistics/statistics.py

@@ -77,6 +77,7 @@ def from_config(cls, config: dict[str, Any]) -> TensorStatistic:
 class MinMaxTensorStatistic(TensorStatistic):
     MIN_STAT: ClassVar[str] = "min_values"
     MAX_STAT: ClassVar[str] = "max_values"
+    MIN_MAX_STAT: ClassVar[str] = "min_max_values"
 
     min_values: Tensor
     max_values: Tensor
@@ -86,6 +87,14 @@ def __eq__(self, other: TensorStatistic):
             return fns.allclose(self.min_values, other.min_values) and fns.allclose(self.max_values, other.max_values)
         return False
 
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> TensorStatistic:
+        if cls.MIN_MAX_STAT in config:
+            # Build MinMaxTensorStatistic for aggregators which
+            # outputs both min and max values from a single aggregator instance.
+            return cls(**config[cls.MIN_MAX_STAT])
+        return cls(min_values=config[cls.MIN_STAT], max_values=config[cls.MAX_STAT])
+
 
 @dataclass
 class AbsMaxTensorStatistic(TensorStatistic):

src/nncf/openvino/statistics/collectors.py

@@ -151,4 +151,5 @@ def get_raw_stat_collector(num_samples: Optional[int] = None) -> TensorCollector
     StatisticsType.MEAN: OVMeanReducer,
     StatisticsType.QUANTILE: OVQuantileReducer,
     StatisticsType.ABS_QUANTILE: OVAbsQuantileReducer,
+    StatisticsType.RAW: RawReducer,
 }

src/nncf/quantization/algorithms/min_max/algorithm.py

@@ -50,6 +50,7 @@
 from nncf.common.utils.backend import BackendType
 from nncf.common.utils.backend import get_backend
 from nncf.experimental.common.tensor_statistics.collectors import AGGREGATORS_MAP
+from nncf.experimental.common.tensor_statistics.collectors import HistogramAggregator
 from nncf.experimental.common.tensor_statistics.collectors import TensorCollector
 from nncf.experimental.common.tensor_statistics.statistics import MinMaxTensorStatistic
 from nncf.parameters import ModelType
@@ -432,7 +433,23 @@ def _get_range_estimator_parameters(
         user_params = self._range_estimator_params[quantizer_group]
         if user_params is None:
             return deepcopy(params)
-
+        if (
+            user_params.min.aggregator_type is AggregatorType.HISTOGRAM
+            or user_params.max.aggregator_type is AggregatorType.HISTOGRAM
+        ):
+            if user_params != RangeEstimatorParametersSet.HISTOGRAM:
+                msg = (
+                    f"Given parameters set {user_params} is not supported by NNCF."
+                    " Please use the RangeEstimatorParametersSet.HISTOGRAM to enable the histogram aggregation."
+                )
+                raise nncf.ParameterNotSupportedError(msg)
+            if quantizer_config.per_channel:
+                msg = (
+                    f"Rollback to RangeEstimatorParametersSet.MINMAX for the target point: '{target_point}' as"
+                    " HistogramAggregator does not support per-channel activations."
+                )
+                nncf_logger.warning(msg)
+                user_params = RangeEstimatorParametersSet.MINMAX
         min_changes = changes_asdict(user_params.min)
         min_statistic_collector = dataclasses.replace(params.min, **min_changes)
 
@@ -493,6 +510,19 @@ def _get_stat_collector(
             num_samples=num_samples,
         )
 
+    def _get_histogram_statistic_collector(self, num_samples: int) -> TensorCollector:
+        """
+        Return the histogram statistic collector.
+
+        :param num_samples: Maximum number of samples to collect.
+        :return: An histogram TensorCollector for the statistics calculation.
+        """
+        reducer = self._backend_entity.reducer_map[StatisticsType.RAW]()
+        aggregator = HistogramAggregator(num_samples=num_samples)
+        collector = TensorCollector(MinMaxTensorStatistic)
+        collector.register_statistic_branch(MinMaxTensorStatistic.MIN_MAX_STAT, reducer, aggregator)
+        return collector
+
     def _get_statistic_collector(
         self,
         range_estimator_params: RangeEstimatorParameters,
@@ -513,6 +543,9 @@ def _get_statistic_collector(
         :param num_samples: Maximum number of samples to collect.
         :return: TensorCollector for the statistics calculation.
         """
+        if range_estimator_params == RangeEstimatorParametersSet.HISTOGRAM:
+            return self._get_histogram_statistic_collector(num_samples)
+
         if not self._backend_entity.supports_inplace_statistics:
             inplace = False
 

src/nncf/quantization/range_estimator.py

@@ -37,6 +37,7 @@ class StatisticsType(Enum):
     QUANTILE = "quantile"
     ABS_QUANTILE = "abs_quantile"
     MEAN = "mean"
+    RAW = "raw"
 
 
 @api()
@@ -59,6 +60,7 @@ class AggregatorType(Enum):
     MEDIAN = "median"
     MEAN_NO_OUTLIERS = "mean_no_outliers"
     MEDIAN_NO_OUTLIERS = "median_no_outliers"
+    HISTOGRAM = "histogram"
 
 
 @api()
@@ -151,3 +153,8 @@ class RangeEstimatorParametersSet:
         min=StatisticsCollectorParameters(statistics_type=StatisticsType.QUANTILE, aggregator_type=AggregatorType.MEAN),
         max=StatisticsCollectorParameters(statistics_type=StatisticsType.QUANTILE, aggregator_type=AggregatorType.MEAN),
     )
+
+    HISTOGRAM = RangeEstimatorParameters(
+        min=StatisticsCollectorParameters(statistics_type=StatisticsType.RAW, aggregator_type=AggregatorType.HISTOGRAM),
+        max=StatisticsCollectorParameters(statistics_type=StatisticsType.RAW, aggregator_type=AggregatorType.HISTOGRAM),
+    )

src/nncf/tensor/functions/__init__.py

@@ -20,21 +20,26 @@
 from nncf.tensor.functions.numeric import as_tensor_like as as_tensor_like
 from nncf.tensor.functions.numeric import astype as astype
 from nncf.tensor.functions.numeric import atleast_1d as atleast_1d
+from nncf.tensor.functions.numeric import bincount as bincount
 from nncf.tensor.functions.numeric import ceil as ceil
 from nncf.tensor.functions.numeric import clip as clip
 from nncf.tensor.functions.numeric import concatenate as concatenate
 from nncf.tensor.functions.numeric import count_nonzero as count_nonzero
+from nncf.tensor.functions.numeric import cumsum as cumsum
 from nncf.tensor.functions.numeric import device as device
 from nncf.tensor.functions.numeric import diag as diag
 from nncf.tensor.functions.numeric import dtype as dtype
 from nncf.tensor.functions.numeric import expand_dims as expand_dims
 from nncf.tensor.functions.numeric import eye as eye
 from nncf.tensor.functions.numeric import finfo as finfo
 from nncf.tensor.functions.numeric import flatten as flatten
+from nncf.tensor.functions.numeric import floor as floor
 from nncf.tensor.functions.numeric import from_numpy as from_numpy
+from nncf.tensor.functions.numeric import histogram as histogram
 from nncf.tensor.functions.numeric import isclose as isclose
 from nncf.tensor.functions.numeric import isempty as isempty
 from nncf.tensor.functions.numeric import item as item
+from nncf.tensor.functions.numeric import linspace as linspace
 from nncf.tensor.functions.numeric import log2 as log2
 from nncf.tensor.functions.numeric import logical_or as logical_or
 from nncf.tensor.functions.numeric import masked_mean as masked_mean
@@ -52,6 +57,7 @@
 from nncf.tensor.functions.numeric import percentile as percentile
 from nncf.tensor.functions.numeric import power as power
 from nncf.tensor.functions.numeric import quantile as quantile
+from nncf.tensor.functions.numeric import repeat as repeat
 from nncf.tensor.functions.numeric import reshape as reshape
 from nncf.tensor.functions.numeric import round as round
 from nncf.tensor.functions.numeric import searchsorted as searchsorted

src/nncf/tensor/functions/numeric.py

@@ -46,6 +46,26 @@ def backend(a: Tensor) -> TensorBackend:
     """
 
 
+@tensor_dispatcher
+def bincount(a: Tensor, *, weights: Optional[Tensor], minlength: int = 0) -> Tensor:
+    """
+    Count number of occurrences of each value in array of non-negative ints.
+
+    The number of bins (of size 1) is one larger than the largest value in x.
+    If minlength is specified, there will be at least this number of bins in the output array
+    (though it will be longer if necessary, depending on the contents of x).
+    Each bin gives the number of occurrences of its index value in x.
+    If weights is specified the input array is weighted by it, i.e.
+    if a value n is found at position i, out[n] += weight[i] instead of out[n] += 1.
+
+    :param a: Input array.
+    :param weight: Weights, array of the same shape as a.
+    :param minlength: A minimum number of bins for the output array.
+    :return: The result of binning the input array.
+        The length of out is equal to max(np.amax(x)+1, minlength).
+    """
+
+
 @tensor_dispatcher
 def squeeze(a: Tensor, axis: Optional[Union[int, tuple[int, ...]]] = None) -> Tensor:
     """
@@ -127,6 +147,20 @@ def dtype(a: Tensor) -> TensorDataType:
     """
 
 
+@tensor_dispatcher
+def repeat(a: Tensor, repeats: Union[int, Tensor], *, axis: Optional[int] = None) -> Tensor:
+    """
+    Repeats elements of a tensor along a specified axis.
+
+    :param a: Input tensor.
+    :param repeats: The number of repetitions for each element.
+        repeats is broadcasted to fit the shape of the given axis.
+    :param axis: The axis along which to repeat values.
+        By default, use the flattened input array, and return a flat output array.
+    :return: A tensor with repeated elements.
+    """
+
+
 @tensor_dispatcher
 def reshape(a: Tensor, shape: T_SHAPE) -> Tensor:
     """
@@ -201,6 +235,23 @@ def count_nonzero(a: Tensor, axis: T_AXIS = None) -> Tensor:
     """
 
 
+@tensor_dispatcher
+def histogram(
+    a: Tensor,
+    bins: int,
+    *,
+    range: Optional[tuple[float, float]] = None,
+) -> Tensor:
+    """
+    Computes a histogram of the values in a tensor.
+
+    :param a:  The input tensor.
+    :param bins: Defines the number of equal-width bins.
+    :param range: Defines the range of the bins. If not provided, range is simply (a.min(), a.max())
+    :return: A 1D Tensor containing the values of the histogram.
+    """
+
+
 @tensor_dispatcher
 def isempty(a: Tensor) -> bool:
     """
@@ -353,6 +404,16 @@ def median(a: Tensor, axis: T_AXIS = None, keepdims: bool = False) -> Tensor:
     """
 
 
+@tensor_dispatcher
+def floor(a: Tensor) -> Tensor:
+    """
+    Return the floor of the input, element-wise.
+
+    :param a: The input tensor.
+    :return: The floor of the input, element-wise.
+    """
+
+
 @tensor_dispatcher
 def round(a: Tensor, decimals: int = 0) -> Tensor:
     """
@@ -490,6 +551,19 @@ def item(a: Tensor) -> Union[int, float, bool]:
     """
 
 
+@tensor_dispatcher
+def cumsum(a: Tensor, axis: int) -> Tensor:
+    """
+    Return the cumulative sum of the elements along a given axis.
+
+    :param a: The input tensor.
+    :param axis: Axis along which the cumulative sum is computed.
+        The default (None) is to compute the cumsum over the flattened array.
+    :return: A new tensor holding the result. The result has the same size as a,
+        and the same shape as a if axis is not None or a is a 1-d array.
+    """
+
+
 @tensor_dispatcher
 def sum(a: Tensor, axis: Optional[Union[int, tuple[int, ...]]] = None, keepdims: bool = False) -> Tensor:
     """
@@ -728,6 +802,31 @@ def eye(
     return Tensor(get_numeric_backend_fn("eye", backend)(n, m, dtype=dtype, device=device))
 
 
+def linspace(
+    start: float,
+    stop: float,
+    num: int,
+    *,
+    backend: TensorBackend,
+    dtype: Optional[TensorDataType] = None,
+    device: Optional[TensorDeviceType] = None,
+) -> Tensor:
+    """
+    Return a tensor filled with evenly spaced numbers over a specified interval.
+
+    :param start: The starting value for the set of points.
+    :param end: The ending value for the set of points.
+    :param num: Number of samples to generate. Must be non-negative.
+    :param backend: The backend type for which the tensor is required.
+    :param dtype: The data type of the returned tensor If dtype is not given, infer the data type
+        from the other input arguments.
+    :param device: The device on which the tensor will be allocated, If device is not given,
+        then the default device is determined by backend.
+    :return: A tensor with num equally spaced samples in the closed interval [start, stop].
+    """
+    return Tensor(get_numeric_backend_fn("linspace", backend)(start, stop, num, dtype=dtype, device=device))
+
+
 def arange(
     start: float,
     end: Optional[float] = None,

src/nncf/tensor/functions/numpy_numeric.py

@@ -62,6 +62,11 @@ def _(a: T_NUMPY) -> TensorBackend:
     return TensorBackend.numpy
 
 
+@numeric.bincount.register
+def _(a: T_NUMPY, *, weights: Optional[T_NUMPY], minlength: int = 0) -> T_NUMPY:
+    return np.bincount(a, weights=weights, minlength=minlength)
+
+
 @numeric.squeeze.register
 def _(a: T_NUMPY, axis: T_AXIS = None) -> T_NUMPY:
     return np.squeeze(a, axis=axis)
@@ -97,6 +102,11 @@ def _(a: T_NUMPY) -> TensorDataType:
     return DTYPE_MAP_REV[np.dtype(a.dtype)]
 
 
+@numeric.repeat.register
+def _(a: T_NUMPY, repeats: Union[int, T_NUMPY_ARRAY], *, axis: Optional[int] = None) -> T_NUMPY:
+    return np.repeat(a, repeats=repeats, axis=axis)
+
+
 @numeric.reshape.register
 def _(a: T_NUMPY, shape: T_SHAPE) -> T_NUMPY:
     return a.reshape(shape)
@@ -133,6 +143,16 @@ def _(a: T_NUMPY, axis: T_AXIS = None) -> T_NUMPY_ARRAY:
     return np.array(np.count_nonzero(a, axis=axis))
 
 
+@numeric.histogram.register
+def _(
+    a: T_NUMPY,
+    bins: int,
+    *,
+    range: Optional[tuple[float, float]] = None,
+) -> T_NUMPY:
+    return np.histogram(a=a, bins=bins, range=range)[0]
+
+
 @numeric.isempty.register
 def _(a: T_NUMPY) -> bool:
     return a.size == 0
@@ -218,6 +238,11 @@ def _(
     return np.array(np.median(a, axis=axis, keepdims=keepdims))  # type: ignore [arg-type]
 
 
+@numeric.floor.register
+def _(a: T_NUMPY) -> T_NUMPY:
+    return np.floor(a)
+
+
 @numeric.round.register
 def _(a: T_NUMPY, decimals: int = 0) -> T_NUMPY_ARRAY:
     return np.round(a, decimals=decimals)
@@ -289,6 +314,11 @@ def _(a: T_NUMPY) -> T_NUMBER:
     return a.item()
 
 
+@numeric.cumsum.register
+def _(a: T_NUMPY, axis: int) -> T_NUMPY:
+    return np.cumsum(a, axis=axis)
+
+
 @numeric.sum.register
 def _(a: T_NUMPY, axis: T_AXIS = None, keepdims: bool = False) -> T_NUMPY_ARRAY:
     return np.array(np.sum(a, axis=axis, keepdims=keepdims))
@@ -420,6 +450,19 @@ def eye(
     return np.eye(n, m, dtype=np_dtype)
 
 
+def linspace(
+    start: float,
+    end: float,
+    num: int,
+    *,
+    dtype: Optional[TensorDataType] = None,
+    device: Optional[TensorDeviceType] = None,
+) -> T_NUMPY_ARRAY:
+    validate_device(device)
+    np_dtype = convert_to_numpy_dtype(dtype)
+    return np.linspace(start, end, num, dtype=np_dtype)
+
+
 def arange(
     start: float,
     end: float,