fix: solve the problem of circular import;

moved the functions of parsing delta to util.py.

pypots/data/__init__.py

@@ -9,16 +9,19 @@
 from pypots.data.dataset_for_brits import DatasetForBRITS
 from pypots.data.dataset_for_grud import DatasetForGRUD
 from pypots.data.dataset_for_mit import DatasetForMIT
+
 from pypots.data.generating import (
     generate_random_walk,
     generate_random_walk_for_classification,
 )
-from pypots.data.integration import (
+
+from pypots.data.utils import (
     masked_fill,
     mcar,
     pickle_load,
     pickle_dump,
 )
+
 from pypots.data.load_specific_datasets import (
     list_supported_datasets,
     load_specific_dataset,

pypots/data/dataset_for_brits.py

@@ -9,7 +9,7 @@
 import torch
 
 from pypots.data.base import BaseDataset
-from pypots.data.dataset_for_grud import torch_parse_delta
+from pypots.data.utils import torch_parse_delta
 
 
 class DatasetForBRITS(BaseDataset):

pypots/data/dataset_for_grud.py

@@ -6,95 +6,15 @@
 # License: GLP-v3
 
 
-import numpy as np
 import torch
 
 from pypots.data.base import BaseDataset
+from pypots.data.utils import torch_parse_delta
 from pypots.imputation.locf import LOCF
 
 
-def torch_parse_delta(missing_mask):
-    """Generate time-gap (delta) matrix from missing masks. Please refer to :cite:`che2018GRUD` for its math definition.
-
-    Parameters
-    ----------
-    missing_mask : torch.tensor, shape of [n_steps, n_features] or [n_samples, n_steps, n_features]
-        Binary masks indicate missing values.
-
-    Returns
-    -------
-    delta, torch.tensor,
-        Delta matrix indicates time gaps of missing values.
-    """
-
-    def cal_delta_for_single_sample(mask):
-        """calculate single sample's delta. The sample's shape is [n_steps, n_features]."""
-        d = []
-        for step in range(n_steps):
-            if step == 0:
-                d.append(torch.zeros(1, n_features, device=device))
-            else:
-                d.append(
-                    torch.ones(1, n_features, device=device) + (1 - mask[step]) * d[-1]
-                )
-        d = torch.concat(d, dim=0)
-        return d
-
-    device = missing_mask.device
-    if len(missing_mask.shape) == 2:
-        n_steps, n_features = missing_mask.shape
-        delta = cal_delta_for_single_sample(missing_mask)
-    else:
-        n_samples, n_steps, n_features = missing_mask.shape
-        delta_collector = []
-        for m_mask in missing_mask:
-            delta = cal_delta_for_single_sample(m_mask)
-            delta_collector.append(delta.unsqueeze(0))
-        delta = torch.concat(delta_collector, dim=0)
-
-    return delta
-
-
-def numpy_parse_delta(missing_mask):
-    """Generate time-gap (delta) matrix from missing masks. Please refer to :cite:`che2018GRUD` for its math definition.
-
-    Parameters
-    ----------
-    missing_mask : np.ndarray, shape of [n_steps, n_features] or [n_samples, n_steps, n_features]
-        Binary masks indicate missing values.
-
-    Returns
-    -------
-    delta, np.ndarray,
-        Delta matrix indicates time gaps of missing values.
-    """
-
-    def cal_delta_for_single_sample(mask):
-        """calculate single sample's delta. The sample's shape is [n_steps, n_features]."""
-        d = []
-        for step in range(seq_len):
-            if step == 0:
-                d.append(np.zeros(n_features))
-            else:
-                d.append(np.ones(n_features) + (1 - mask[step]) * d[-1])
-        d = np.asarray(d)
-        return d
-
-    if len(missing_mask.shape) == 2:
-        seq_len, n_features = missing_mask.shape
-        delta = cal_delta_for_single_sample(missing_mask)
-    else:
-        n_samples, seq_len, n_features = missing_mask.shape
-        delta_collector = []
-        for m_mask in missing_mask:
-            delta = cal_delta_for_single_sample(m_mask)
-            delta_collector.append(delta)
-        delta = np.asarray(delta_collector)
-    return delta
-
-
 class DatasetForGRUD(BaseDataset):
-    """Dataset class for model GRUD.
+    """Dataset class for model GRU-D.
 
     Parameters
     ----------
@@ -113,7 +33,6 @@ class DatasetForGRUD(BaseDataset):
 
     def __init__(self, data, file_type="h5py"):
         super().__init__(data, file_type)
-
         self.locf = LOCF()
 
         if not isinstance(self.data, str):  # data from array

pypots/data/utils.py

@@ -0,0 +1,111 @@
+"""
+Data utils.
+"""
+
+# Created by Wenjie Du <wenjay.du@gmail.com>
+# License: GLP-v3
+
+import numpy as np
+import torch
+
+
+import pycorruptor as corruptor
+from tsdb import (
+    pickle_load as _pickle_load,
+    pickle_dump as _pickle_dump,
+)
+
+pickle_load = _pickle_load
+pickle_dump = _pickle_dump
+
+
+def cal_missing_rate(X):
+    return corruptor.cal_missing_rate(X)
+
+
+def masked_fill(X, mask, val):
+    return corruptor.masked_fill(X, mask, val)
+
+
+def mcar(X, rate, nan=0):
+    return corruptor.mcar(X, rate, nan)
+
+
+def torch_parse_delta(missing_mask):
+    """Generate time-gap (delta) matrix from missing masks. Please refer to :cite:`che2018GRUD` for its math definition.
+
+    Parameters
+    ----------
+    missing_mask : torch.tensor, shape of [n_steps, n_features] or [n_samples, n_steps, n_features]
+        Binary masks indicate missing values.
+
+    Returns
+    -------
+    delta, torch.tensor,
+        Delta matrix indicates time gaps of missing values.
+    """
+
+    def cal_delta_for_single_sample(mask):
+        """calculate single sample's delta. The sample's shape is [n_steps, n_features]."""
+        d = []
+        for step in range(n_steps):
+            if step == 0:
+                d.append(torch.zeros(1, n_features, device=device))
+            else:
+                d.append(
+                    torch.ones(1, n_features, device=device) + (1 - mask[step]) * d[-1]
+                )
+        d = torch.concat(d, dim=0)
+        return d
+
+    device = missing_mask.device
+    if len(missing_mask.shape) == 2:
+        n_steps, n_features = missing_mask.shape
+        delta = cal_delta_for_single_sample(missing_mask)
+    else:
+        n_samples, n_steps, n_features = missing_mask.shape
+        delta_collector = []
+        for m_mask in missing_mask:
+            delta = cal_delta_for_single_sample(m_mask)
+            delta_collector.append(delta.unsqueeze(0))
+        delta = torch.concat(delta_collector, dim=0)
+
+    return delta
+
+
+def numpy_parse_delta(missing_mask):
+    """Generate time-gap (delta) matrix from missing masks. Please refer to :cite:`che2018GRUD` for its math definition.
+
+    Parameters
+    ----------
+    missing_mask : np.ndarray, shape of [n_steps, n_features] or [n_samples, n_steps, n_features]
+        Binary masks indicate missing values.
+
+    Returns
+    -------
+    delta, np.ndarray,
+        Delta matrix indicates time gaps of missing values.
+    """
+
+    def cal_delta_for_single_sample(mask):
+        """calculate single sample's delta. The sample's shape is [n_steps, n_features]."""
+        d = []
+        for step in range(seq_len):
+            if step == 0:
+                d.append(np.zeros(n_features))
+            else:
+                d.append(np.ones(n_features) + (1 - mask[step]) * d[-1])
+        d = np.asarray(d)
+        return d
+
+    if len(missing_mask.shape) == 2:
+        seq_len, n_features = missing_mask.shape
+        delta = cal_delta_for_single_sample(missing_mask)
+    else:
+        n_samples, seq_len, n_features = missing_mask.shape
+        delta_collector = []
+        for m_mask in missing_mask:
+            delta = cal_delta_for_single_sample(m_mask)
+            delta_collector.append(delta)
+        delta = np.asarray(delta_collector)
+    return delta

pypots/imputation/brits.py

@@ -1,7 +1,11 @@
 """
 PyTorch BRITS model for the time-series imputation task.
-Some part of the code is from https://github.com/caow13/BRITS.
+
+Notes
+-----
+Partial implementation uses code from https://github.com/caow13/BRITS.
 """
+
 # Created by Wenjie Du <wenjay.du@gmail.com>
 # License: GPL-v3
 
@@ -16,7 +20,7 @@
 from torch.utils.data import DataLoader
 
 from pypots.data.dataset_for_brits import DatasetForBRITS
-from pypots.data.integration import mcar, masked_fill
+from pypots.data.utils import mcar, masked_fill
 from pypots.imputation.base import BaseNNImputer
 from pypots.utils.metrics import cal_mae
 

pypots/imputation/saits.py

@@ -1,6 +1,9 @@
 """
 PyTorch SAITS model for the time-series imputation task.
-Some part of the code is from https://github.com/WenjieDu/SAITS.
+
+Notes
+-----
+Partial implementation uses code from https://github.com/WenjieDu/SAITS.
 """
 
 # Created by Wenjie Du <wenjay.du@gmail.com>
@@ -14,7 +17,7 @@
 
 from pypots.data.base import BaseDataset
 from pypots.data.dataset_for_mit import DatasetForMIT
-from pypots.data.integration import mcar, masked_fill
+from pypots.data.utils import mcar, masked_fill
 from pypots.imputation.base import BaseNNImputer
 from pypots.imputation.transformer import EncoderLayer, PositionalEncoding
 from pypots.utils.metrics import cal_mae

pypots/imputation/transformer.py

@@ -1,6 +1,9 @@
 """
 PyTorch Transformer model for the time-series imputation task.
-Some part of the code is from https://github.com/WenjieDu/SAITS.
+
+Notes
+-----
+Partial implementation uses code from https://github.com/WenjieDu/SAITS.
 """
 
 # Created by Wenjie Du <wenjay.du@gmail.com>
@@ -14,7 +17,7 @@
 
 from pypots.data.base import BaseDataset
 from pypots.data.dataset_for_mit import DatasetForMIT
-from pypots.data.integration import mcar, masked_fill
+from pypots.data.utils import mcar, masked_fill
 from pypots.imputation.base import BaseNNImputer
 from pypots.utils.metrics import cal_mae