chore(dataset): remove dataset origin field (#2020)

remove dataset origin field
star-whale · Mar 30, 2023 · 9adab60 · 9adab60
1 parent 3158e86
commit 9adab60
Show file tree

Hide file tree

Showing 6 changed files with 4 additions and 45 deletions.
diff --git a/client/starwhale/base/type.py b/client/starwhale/base/type.py
@@ -89,12 +89,6 @@ class ObjectStoreType(Enum):
     UNDEFINED = "undefined"
 
 
-@unique
-class DataOriginType(Enum):
-    NEW = "+"
-    INHERIT = "~"
-
-
 @unique
 class DependencyType(Enum):
     PIP_PKG = "pip_pkg"

diff --git a/client/starwhale/core/dataset/tabular.py b/client/starwhale/core/dataset/tabular.py
@@ -18,13 +18,7 @@
 from starwhale.utils import validate_obj_name
 from starwhale.consts import ENV_POD_NAME, STANDALONE_INSTANCE
 from starwhale.base.uri import URI
-from starwhale.base.type import (
-    URIType,
-    InstanceType,
-    DataFormatType,
-    DataOriginType,
-    ObjectStoreType,
-)
+from starwhale.base.type import URIType, InstanceType, DataFormatType, ObjectStoreType
 from starwhale.base.mixin import ASDictMixin, _do_asdict_convert
 from starwhale.consts.env import SWEnv
 from starwhale.utils.error import (
@@ -102,12 +96,10 @@ class TabularDatasetRow(ASDictMixin):
     def __init__(
         self,
         id: t.Union[str, int],
-        origin: DataOriginType = DataOriginType.NEW,
         features: t.Optional[t.Dict[str, t.Any]] = None,
         **kw: t.Union[str, int, float],
     ) -> None:
         self.id = id
-        self.origin = origin
         self.features = features or {}
         self.extra_kw = kw
         # TODO: add non-starwhale object store related fields, such as address, authority
@@ -118,7 +110,6 @@ def __init__(
     def from_datastore(
         cls,
         id: t.Union[str, int],
-        origin: str = DataOriginType.NEW.value,
         **kw: t.Any,
     ) -> TabularDatasetRow:
         _content = {}
@@ -132,17 +123,13 @@ def from_datastore(
 
         return cls(
             id=id,
-            origin=DataOriginType(origin),
             features=_content,
             **_extra_kw,
         )
 
     def __eq__(self, o: object) -> bool:
         s = deepcopy(self.__dict__)
         o = deepcopy(o.__dict__)
-
-        s.pop("origin", None)
-        o.pop("origin", None)
         return s == o
 
     def _do_validate(self) -> None:
@@ -155,9 +142,6 @@ def _do_validate(self) -> None:
         if not isinstance(self.features, dict) or not self.features:
             raise FieldTypeOrValueError("no data field")
 
-        if not isinstance(self.origin, DataOriginType):
-            raise NoSupportError(f"data origin: {self.origin}")
-
     def __str__(self) -> str:
         return f"row-{self.id}"
 
@@ -192,7 +176,6 @@ def artifacts(self) -> t.List[BaseArtifact]:
 class TabularDataset:
     _map_types = {
         "data_format": DataFormatType,
-        "origin": DataOriginType,
         "object_store_type": ObjectStoreType,
     }
 

diff --git a/client/tests/core/test_dataset.py b/client/tests/core/test_dataset.py
@@ -21,7 +21,7 @@
 from starwhale.base.uri import URI
 from starwhale.utils.fs import ensure_dir, ensure_file
 from starwhale.api._impl import data_store
-from starwhale.base.type import URIType, BundleType, DataOriginType
+from starwhale.base.type import URIType, BundleType
 from starwhale.utils.config import SWCliConfigMixed
 from starwhale.core.dataset.cli import _list as list_cli
 from starwhale.core.dataset.cli import _build as build_cli
@@ -301,7 +301,6 @@ def test_head(
                     ),
                     "label": 0,
                 },
-                origin=DataOriginType.NEW,
             ),
             TabularDatasetRow(
                 id="label-1",
@@ -317,7 +316,6 @@ def test_head(
                     ),
                     "label": 1,
                 },
-                origin=DataOriginType.NEW,
             ),
         ]
         content = b"\x00_\xfe\xc3\x00\x00\x00\x00"

diff --git a/client/tests/sdk/test_dataset.py b/client/tests/sdk/test_dataset.py
@@ -342,7 +342,6 @@ def test_download(self, rm: Mocker) -> None:
                                 },
                             ],
                         },
-                        {"type": "STRING", "name": "origin"},
                         {"type": "INT64", "name": "_append_seq_id"},
                         {
                             "type": "OBJECT",
@@ -374,7 +373,6 @@ def test_download(self, rm: Mocker) -> None:
                                 "width": "0000000000000003",
                                 "height": "0000000000000004",
                             },
-                            "origin": "+",
                         }
                     ],
                 }
@@ -1155,7 +1153,6 @@ def test_row(self) -> None:
             "id": 0,
             "features/l": Image(link=Link("abcdef"), shape=[1, 2, 3]),
             "features/a": 1,
-            "origin": "+",
         }
 
         u_row_dict = u_row.asdict()

diff --git a/client/tests/sdk/test_evaluation.py b/client/tests/sdk/test_evaluation.py
@@ -14,7 +14,7 @@
 from starwhale.consts import DEFAULT_PROJECT
 from starwhale.base.uri import URI
 from starwhale.utils.fs import ensure_dir, ensure_file
-from starwhale.base.type import URIType, RunSubDirType, DataOriginType
+from starwhale.base.type import URIType, RunSubDirType
 from starwhale.utils.error import ParameterError
 from starwhale.core.eval.store import EvaluationStorage
 from starwhale.core.job.context import Context
@@ -263,7 +263,6 @@ def cmp(self, _data_loader: t.Any) -> t.Any:
                         ),
                         "label": label_data,
                     },
-                    origin=DataOriginType.NEW,
                     id=0,
                 ).asdict(),
             ],

diff --git a/client/tests/sdk/test_loader.py b/client/tests/sdk/test_loader.py
@@ -13,7 +13,7 @@
 from starwhale.consts import HTTPMethod, SWDSBackendType
 from starwhale.base.uri import URI
 from starwhale.utils.fs import ensure_dir
-from starwhale.base.type import URIType, DataOriginType
+from starwhale.base.type import URIType
 from starwhale.consts.env import SWEnv
 from starwhale.utils.error import ParameterError
 from starwhale.core.dataset.type import Link, Image, DatasetSummary, GrayscaleImage
@@ -99,7 +99,6 @@ def test_user_raw_local_store(
                     ),
                     "label": 0,
                 },
-                origin=DataOriginType.NEW,
                 id="path/0",
             )
         ]
@@ -220,7 +219,6 @@ def test_user_raw_remote_store(
                         ),
                         "label": 0,
                     },
-                    origin=DataOriginType.NEW,
                     id=0,
                 ),
                 TabularDatasetRow(
@@ -234,7 +232,6 @@ def test_user_raw_remote_store(
                         ),
                         "label": 1,
                     },
-                    origin=DataOriginType.NEW,
                     id=1,
                 ),
                 TabularDatasetRow(
@@ -248,7 +245,6 @@ def test_user_raw_remote_store(
                         ),
                         "label": 1,
                     },
-                    origin=DataOriginType.NEW,
                     id=2,
                 ),
                 TabularDatasetRow(
@@ -262,7 +258,6 @@ def test_user_raw_remote_store(
                         ),
                         "label": 1,
                     },
-                    origin=DataOriginType.NEW,
                     id=3,
                 ),
             ]
@@ -366,7 +361,6 @@ def test_swds_bin_s3(
                     ),
                     "label": 0,
                 },
-                origin=DataOriginType.NEW,
                 id=0,
             )
         ]
@@ -443,7 +437,6 @@ def test_swds_bin_local_fs(self, m_scan: MagicMock, m_summary: MagicMock) -> Non
                     ),
                     "label": 0,
                 },
-                origin=DataOriginType.NEW,
                 id=0,
             ),
             TabularDatasetRow(
@@ -459,7 +452,6 @@ def test_swds_bin_local_fs(self, m_scan: MagicMock, m_summary: MagicMock) -> Non
                     ),
                     "label": 1,
                 },
-                origin=DataOriginType.NEW,
                 id=1,
             ),
         ]
@@ -544,7 +536,6 @@ def test_remote_batch_sign(
                             )
                         ),
                     },
-                    origin=DataOriginType.NEW,
                 ),
                 TabularDatasetRow(
                     id="b",
@@ -568,7 +559,6 @@ def test_remote_batch_sign(
                             )
                         ),
                     },
-                    origin=DataOriginType.NEW,
                 ),
             ],
             [
@@ -594,7 +584,6 @@ def test_remote_batch_sign(
                             )
                         ),
                     },
-                    origin=DataOriginType.NEW,
                 ),
                 TabularDatasetRow(
                     id="d",
@@ -618,7 +607,6 @@ def test_remote_batch_sign(
                             )
                         ),
                     },
-                    origin=DataOriginType.NEW,
                 ),
             ],
         ]