Use PathManager for io

Summary: Update io to use manifold instead of gluster

Reviewed By: vaibhava0

Differential Revision: D23854788

fbshipit-source-id: bbe2fef9e50657e521addf85e5fb297f228817ab

dev/test_models.py

@@ -16,12 +16,13 @@
 
 import pycls.core.builders as builders
 import pycls.core.distributed as dist
+import pycls.core.env as env
 import pycls.core.logging as logging
 import pycls.core.net as net
 import pycls.core.trainer as trainer
 import pycls.models.model_zoo as model_zoo
 from parameterized import parameterized
-from pycls.core.config import cfg, reset_cfg
+from pycls.core.config import cfg, merge_from_file, reset_cfg
 
 
 # Location of pycls directory
@@ -37,14 +38,14 @@ def test_complexity(key):
     """Measure the complexity of a single model."""
     reset_cfg()
     cfg_file = os.path.join(_PYCLS_DIR, key)
-    cfg.merge_from_file(cfg_file)
+    merge_from_file(cfg_file)
     return net.complexity(builders.get_model())
 
 
 def test_timing(key):
     """Measure the timing of a single model."""
     reset_cfg()
-    cfg.merge_from_file(model_zoo.get_config_file(key))
+    merge_from_file(model_zoo.get_config_file(key))
     cfg.PREC_TIME.WARMUP_ITER, cfg.PREC_TIME.NUM_ITER = 5, 50
     cfg.OUT_DIR, cfg.LOG_DEST = tempfile.mkdtemp(), "file"
     dist.multi_proc_run(num_proc=cfg.NUM_GPUS, fun=trainer.time_model)
@@ -57,7 +58,7 @@ def test_timing(key):
 def test_error(key):
     """Measure the error of a single model."""
     reset_cfg()
-    cfg.merge_from_file(model_zoo.get_config_file(key))
+    merge_from_file(model_zoo.get_config_file(key))
     cfg.TEST.WEIGHTS = model_zoo.get_weights_file(key)
     cfg.OUT_DIR, cfg.LOG_DEST = tempfile.mkdtemp(), "file"
     dist.multi_proc_run(num_proc=cfg.NUM_GPUS, fun=trainer.test_model)
@@ -137,6 +138,7 @@ class TestError(unittest.TestCase):
     @parameterized.expand(parse_tests(load_test_data("error")), skip_on_empty=True)
     @unittest.skipIf(not _RUN_ERROR_TESTS, "Skipping error tests")
     def test(self, key, out_expected):
+        env.setup_env()
         print("\nTesting error of: {}".format(key))
         out = test_error(key)
         print("expected = {}".format(out_expected))

pycls/core/io.py

@@ -13,6 +13,8 @@
 import sys
 from urllib import request as urlrequest
 
+from iopath.common.file_io import g_pathmgr
+
 
 logger = logging.getLogger(__name__)
 
@@ -29,11 +31,11 @@ def cache_url(url_or_file, cache_dir, base_url=_PYCLS_BASE_URL):
     url = url_or_file
     assert url.startswith(base_url), "url must start with: {}".format(base_url)
     cache_file_path = url.replace(base_url, cache_dir)
-    if os.path.exists(cache_file_path):
+    if g_pathmgr.exists(cache_file_path):
         return cache_file_path
     cache_file_dir = os.path.dirname(cache_file_path)
-    if not os.path.exists(cache_file_dir):
-        os.makedirs(cache_file_dir)
+    if not g_pathmgr.exists(cache_file_dir):
+        g_pathmgr.mkdirs(cache_file_dir)
     logger.info("Downloading remote file {} to {}".format(url, cache_file_path))
     download_url(url, cache_file_path)
     return cache_file_path
@@ -64,7 +66,7 @@ def download_url(url, dst_file_path, chunk_size=8192, progress_hook=_progress_ba
     total_size = response.info().get("Content-Length").strip()
     total_size = int(total_size)
     bytes_so_far = 0
-    with open(dst_file_path, "wb") as f:
+    with g_pathmgr.open(dst_file_path, "wb") as f:
         while 1:
             chunk = response.read(chunk_size)
             bytes_so_far += len(chunk)