From 87e3c31a660c4593564ad06860513242bacbaa69 Mon Sep 17 00:00:00 2001 From: Dave Schnizlein Date: Wed, 13 Jan 2021 10:53:04 -0800 Subject: [PATCH] Use PathManager for io Summary: Update io to use manifold instead of gluster Reviewed By: vaibhava0 Differential Revision: D23854788 fbshipit-source-id: bbe2fef9e50657e521addf85e5fb297f228817ab --- dev/test_models.py | 10 ++++++---- pycls/core/io.py | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/dev/test_models.py b/dev/test_models.py index 5f1df0b..53d4cd2 100755 --- a/dev/test_models.py +++ b/dev/test_models.py @@ -16,12 +16,13 @@ import pycls.core.builders as builders import pycls.core.distributed as dist +import pycls.core.env as env import pycls.core.logging as logging import pycls.core.net as net import pycls.core.trainer as trainer import pycls.models.model_zoo as model_zoo from parameterized import parameterized -from pycls.core.config import cfg, reset_cfg +from pycls.core.config import cfg, merge_from_file, reset_cfg # Location of pycls directory @@ -37,14 +38,14 @@ def test_complexity(key): """Measure the complexity of a single model.""" reset_cfg() cfg_file = os.path.join(_PYCLS_DIR, key) - cfg.merge_from_file(cfg_file) + merge_from_file(cfg_file) return net.complexity(builders.get_model()) def test_timing(key): """Measure the timing of a single model.""" reset_cfg() - cfg.merge_from_file(model_zoo.get_config_file(key)) + merge_from_file(model_zoo.get_config_file(key)) cfg.PREC_TIME.WARMUP_ITER, cfg.PREC_TIME.NUM_ITER = 5, 50 cfg.OUT_DIR, cfg.LOG_DEST = tempfile.mkdtemp(), "file" dist.multi_proc_run(num_proc=cfg.NUM_GPUS, fun=trainer.time_model) @@ -57,7 +58,7 @@ def test_timing(key): def test_error(key): """Measure the error of a single model.""" reset_cfg() - cfg.merge_from_file(model_zoo.get_config_file(key)) + merge_from_file(model_zoo.get_config_file(key)) cfg.TEST.WEIGHTS = model_zoo.get_weights_file(key) cfg.OUT_DIR, cfg.LOG_DEST = tempfile.mkdtemp(), "file" dist.multi_proc_run(num_proc=cfg.NUM_GPUS, fun=trainer.test_model) @@ -137,6 +138,7 @@ class TestError(unittest.TestCase): @parameterized.expand(parse_tests(load_test_data("error")), skip_on_empty=True) @unittest.skipIf(not _RUN_ERROR_TESTS, "Skipping error tests") def test(self, key, out_expected): + env.setup_env() print("\nTesting error of: {}".format(key)) out = test_error(key) print("expected = {}".format(out_expected)) diff --git a/pycls/core/io.py b/pycls/core/io.py index ae5049e..0f19a37 100644 --- a/pycls/core/io.py +++ b/pycls/core/io.py @@ -13,6 +13,8 @@ import sys from urllib import request as urlrequest +from iopath.common.file_io import g_pathmgr + logger = logging.getLogger(__name__) @@ -29,11 +31,11 @@ def cache_url(url_or_file, cache_dir, base_url=_PYCLS_BASE_URL): url = url_or_file assert url.startswith(base_url), "url must start with: {}".format(base_url) cache_file_path = url.replace(base_url, cache_dir) - if os.path.exists(cache_file_path): + if g_pathmgr.exists(cache_file_path): return cache_file_path cache_file_dir = os.path.dirname(cache_file_path) - if not os.path.exists(cache_file_dir): - os.makedirs(cache_file_dir) + if not g_pathmgr.exists(cache_file_dir): + g_pathmgr.mkdirs(cache_file_dir) logger.info("Downloading remote file {} to {}".format(url, cache_file_path)) download_url(url, cache_file_path) return cache_file_path @@ -64,7 +66,7 @@ def download_url(url, dst_file_path, chunk_size=8192, progress_hook=_progress_ba total_size = response.info().get("Content-Length").strip() total_size = int(total_size) bytes_so_far = 0 - with open(dst_file_path, "wb") as f: + with g_pathmgr.open(dst_file_path, "wb") as f: while 1: chunk = response.read(chunk_size) bytes_so_far += len(chunk)