chore(client): make client ut stable

client/Makefile

@@ -59,7 +59,7 @@ ci-isort:
 .POHNY: ut
 ut:
 	echo "ut"
-	pytest tests -vvrfEsx --cov-config=.coveragerc --cov=starwhale --cov-report=xml:coverage.xml --cov-report=term-missing
+	python -m pytest tests -vvrfEsx --cov-config=.coveragerc --cov=starwhale --cov-report=xml:coverage.xml --cov-report=term-missing
 
 .POHNY: all-check
 all-check: ci-format-checker ci-lint ci-mypy ci-isort ut

client/starwhale/api/_impl/dataset/builder.py

@@ -1,3 +1,4 @@
+import os
 import struct
 import typing as t
 import tempfile
@@ -250,7 +251,13 @@ def make_swds(self) -> DatasetSummary:
             increased_rows += 1
 
         try:
+            empty = dwriter.tell() == 0
             dwriter.close()
+            if empty:
+                # last file is empty
+                f = ds_copy_candidates[fno]
+                del ds_copy_candidates[fno]
+                os.unlink(f)
         except Exception as e:
             print(f"data write close exception: {e}")
 

client/tests/base/test_copy.py

@@ -4,6 +4,7 @@
 from requests_mock import Mocker
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import get_predefined_config_yaml
 from starwhale.utils import config as sw_config
 from starwhale.consts import (
     HTTPMethod,
@@ -20,8 +21,6 @@
 from starwhale.base.bundle_copy import BundleCopy
 from starwhale.core.dataset.store import DatasetStorage
 
-from .. import get_predefined_config_yaml
-
 _existed_config_contents = get_predefined_config_yaml()
 
 

client/tests/base/test_uri.py

@@ -1,12 +1,11 @@
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import get_predefined_config_yaml
 from starwhale.utils import config as sw_config
 from starwhale.base.uri import URI
 from starwhale.base.type import URIType, InstanceType
 from starwhale.utils.config import get_swcli_config_path
 
-from .. import get_predefined_config_yaml
-
 _existed_config_contents = get_predefined_config_yaml()
 
 

client/tests/core/test_dataset.py

@@ -4,6 +4,7 @@
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR
 from starwhale.utils import config as sw_config
 from starwhale.utils import load_yaml
 from starwhale.consts import (
@@ -19,8 +20,6 @@
 from starwhale.core.dataset.view import DatasetTermView
 from starwhale.core.dataset.model import StandaloneDataset
 
-from .. import ROOT_DIR
-
 _dataset_data_dir = f"{ROOT_DIR}/data/dataset"
 _dataset_yaml = open(f"{_dataset_data_dir}/dataset.yaml").read()
 

client/tests/core/test_eval.py

@@ -5,6 +5,7 @@
 from requests_mock import Mocker
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR, get_predefined_config_yaml
 from starwhale.utils import config as sw_config
 from starwhale.consts import HTTPMethod, RECOVER_DIRNAME, DEFAULT_MANIFEST_NAME
 from starwhale.base.uri import URI
@@ -14,8 +15,6 @@
 from starwhale.core.eval.model import CloudEvaluationJob, StandaloneEvaluationJob
 from starwhale.core.eval.store import EvaluationStorage
 
-from .. import ROOT_DIR, get_predefined_config_yaml
-
 _job_data_dir = f"{ROOT_DIR}/data/job"
 _job_manifest = open(f"{_job_data_dir}/job_manifest.yaml").read()
 _job_list = open(f"{_job_data_dir}/job_list_resp.json").read()

client/tests/core/test_executor.py

@@ -3,6 +3,7 @@
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR
 from starwhale.utils import config as sw_config
 from starwhale.utils import load_yaml
 from starwhale.consts import (
@@ -17,8 +18,6 @@
 from starwhale.utils.config import SWCliConfigMixed
 from starwhale.core.eval.executor import EvalExecutor
 
-from .. import ROOT_DIR
-
 _dataset_manifest = open(f"{ROOT_DIR}/data/dataset.yaml").read()
 _model_data_dir = f"{ROOT_DIR}/data/model"
 _model_yaml = open(f"{_model_data_dir}/model.yaml").read()

client/tests/core/test_instance.py

@@ -1,13 +1,12 @@
 from requests_mock import Mocker
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import get_predefined_config_yaml
 from starwhale.utils import config as sw_config
 from starwhale.consts import HTTPMethod
 from starwhale.utils.config import SWCliConfigMixed, get_swcli_config_path
 from starwhale.core.instance.view import InstanceTermView
 
-from .. import get_predefined_config_yaml
-
 _existed_config_contents = get_predefined_config_yaml()
 
 

client/tests/core/test_job.py

@@ -7,6 +7,7 @@
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR
 from starwhale.consts import DEFAULT_EVALUATION_PIPELINE, DEFAULT_EVALUATION_JOBS_FNAME
 from starwhale.utils.fs import ensure_dir
 from starwhale.api._impl.job import Parser, Context
@@ -21,8 +22,6 @@
 )
 from starwhale.core.job.scheduler import Scheduler
 
-from .. import ROOT_DIR
-
 _job_data_dir = f"{ROOT_DIR}/data/job"
 
 

client/tests/core/test_model.py

@@ -6,6 +6,7 @@
 from requests_mock import Mocker
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR
 from starwhale.utils import config as sw_config
 from starwhale.utils import load_yaml
 from starwhale.consts import (
@@ -26,8 +27,6 @@
 from starwhale.core.model.model import StandaloneModel
 from starwhale.core.instance.view import InstanceTermView
 
-from .. import ROOT_DIR
-
 _model_data_dir = f"{ROOT_DIR}/data/model"
 _model_yaml = open(f"{_model_data_dir}/model.yaml").read()
 

client/tests/core/test_project.py

@@ -3,6 +3,7 @@
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import get_predefined_config_yaml
 from starwhale.utils import config as sw_config
 from starwhale.consts import DEFAULT_PROJECT
 from starwhale.base.uri import URI
@@ -15,8 +16,6 @@
 from starwhale.core.instance.view import InstanceTermView
 from starwhale.core.project.model import StandaloneProject
 
-from .. import get_predefined_config_yaml
-
 _existed_config_contents = get_predefined_config_yaml()
 
 

client/tests/sdk/test_dataset.py

@@ -8,6 +8,7 @@
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR
 from starwhale import Link, MIMEType, UserRawBuildExecutor
 from starwhale.consts import OBJECT_STORE_DIRNAME
 from starwhale.utils.fs import blake2b_file
@@ -34,7 +35,6 @@
     SWDSBinBuildExecutor,
 )
 
-from .. import ROOT_DIR
 from .test_base import BaseTestCase
 
 _mnist_dir = Path(f"{ROOT_DIR}/data/dataset/mnist")
@@ -164,7 +164,7 @@ def test_swds_bin_workflow(self) -> None:
         assert not summary.include_user_raw
         assert not summary.include_link
 
-        assert len(data_files_sign) == 11
+        assert len(data_files_sign) == 10
 
         for _sign in data_files_sign:
             _sign_fpath = Path(self.object_store_dir) / _sign[:2] / _sign

client/tests/sdk/test_loader.py

@@ -4,6 +4,7 @@
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR
 from starwhale import (
     MIMEType,
     S3LinkAuth,
@@ -23,8 +24,6 @@
 )
 from starwhale.core.dataset.tabular import TabularDatasetRow
 
-from .. import ROOT_DIR
-
 
 class TestDataLoader(TestCase):
     def setUp(self) -> None:

client/tests/sdk/test_model.py

@@ -1,12 +1,14 @@
 import os
+import sys
+import errno
 import shutil
 import typing as t
-import sysconfig
 from pathlib import Path
 from unittest.mock import patch, MagicMock
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import ROOT_DIR
 from starwhale import Context, get_data_loader, PipelineHandler, UserRawDataLoader
 from starwhale.consts import DEFAULT_PROJECT
 from starwhale.base.uri import URI
@@ -23,8 +25,6 @@
 from starwhale.core.dataset.store import DatasetStorage
 from starwhale.core.dataset.tabular import TabularDatasetRow
 
-from .. import ROOT_DIR
-
 
 class SimpleHandler(PipelineHandler):
     def ppl(self, data: bytes, **kw: t.Any) -> t.Any:
@@ -217,7 +217,15 @@ def test_ppl(
     @patch("starwhale.api._impl.dataset.loader.TabularDataset.scan")
     @patch("starwhale.core.dataset.model.StandaloneDataset.summary")
     def test_deserializer(self, m_summary: MagicMock, m_scan: MagicMock) -> None:
-        self.fs.add_real_directory(sysconfig.get_paths()["purelib"])
+        # make torch happy
+        for i in sys.path:
+            if not i:
+                continue
+            try:
+                self.fs.add_real_directory(i)
+            except OSError as e:
+                if e.errno not in [errno.EEXIST, errno.ENOENT]:
+                    raise e
         import numpy as np
         import torch
 

client/tests/utils/test_config.py

@@ -2,6 +2,7 @@
 
 from pyfakefs.fake_filesystem_unittest import TestCase
 
+from tests import get_predefined_config_yaml
 from starwhale.utils import config as sw_config
 from starwhale.utils.error import NotFoundError
 from starwhale.utils.config import (
@@ -13,8 +14,6 @@
     get_swcli_config_path,
 )
 
-from .. import get_predefined_config_yaml
-
 _existed_config_contents = get_predefined_config_yaml()