Fix CI errors and add reload when detecting pytorch version inconsist…

…ency (pytorch#1176)

Summary:
- Upgrade the FAMBench versions
- Fix CI errors by adding the functorch dependency

Pull Request resolved: pytorch#1176

Reviewed By: erichan1

Differential Revision: D39580030

Pulled By: xuzhao9

fbshipit-source-id: 5e861e0fbb67c1595f74eef97d874bbb1a26373d

.circleci/config.yml

@@ -18,7 +18,7 @@ jobs:
           command: ./scripts/install_nightlies.sh
       - run:
           name: Setup benchmark suite dependencies
-          command: . ~/miniconda3/etc/profile.d/conda.sh; conda activate base; python install.py
+          command: . ~/miniconda3/etc/profile.d/conda.sh; conda activate base; python install.py --test-mode
       - run:
           name: Validate benchmark components
           command: |

install.py

@@ -77,6 +77,7 @@ def pip_install_requirements(requirements_txt="requirements.txt"):
     parser = argparse.ArgumentParser()
     parser.add_argument("models", nargs='*', default=[],
                         help="Specify one or more models to install. If not set, install all models.")
+    parser.add_argument("--test-mode", action="store_true", help="Run in test mode and check package versions")
     parser.add_argument("--continue_on_fail", action="store_true")
     parser.add_argument("--verbose", "-v", action="store_true")
     parser.add_argument("--component", choices=["distributed"], help="Install requirements for optional components.")
@@ -125,7 +126,7 @@ def pip_install_requirements(requirements_txt="requirements.txt"):
                 Before: {versions}, after: {new_versions}")
         sys.exit(-1)
     from torchbenchmark import setup
-    success &= setup(models=args.models, verbose=args.verbose, continue_on_fail=args.continue_on_fail)
+    success &= setup(models=args.models, verbose=args.verbose, continue_on_fail=args.continue_on_fail, test_mode=args.test_mode)
     if not success:
         if args.continue_on_fail:
             print("Warning: some benchmarks were not installed due to failure")

torchbenchmark/__init__.py

@@ -19,10 +19,21 @@
 REPO_PATH = Path(os.path.abspath(__file__)).parent.parent
 DATA_PATH = os.path.join(REPO_PATH, "torchbenchmark", "data", ".data")
 
-TORCH_DEPS = ['torch', 'torchvision', 'torchtext']
-proxy_suggestion = "Unable to verify https connectivity, " \
-                   "required for setup.\n" \
-                   "Do you need to use a proxy?"
+class add_path():
+    def __init__(self, path):
+        self.path = path
+
+    def __enter__(self):
+        sys.path.insert(0, self.path)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        try:
+            sys.path.remove(self.path)
+        except ValueError:
+            pass
+
+with add_path(str(REPO_PATH)):
+    from utils import TORCH_DEPS, get_pkg_versions, proxy_suggestion
 
 this_dir = pathlib.Path(__file__).parent.absolute()
 model_dir = 'models'
@@ -100,7 +111,7 @@ def _is_internal_model(model_name: str) -> bool:
         return True
     return False
 
-def setup(models: List[str] = [], verbose: bool = True, continue_on_fail: bool = False) -> bool:
+def setup(models: List[str] = [], verbose: bool = True, continue_on_fail: bool = False, test_mode: bool = False) -> bool:
     if not _test_https():
         print(proxy_suggestion)
         sys.exit(-1)
@@ -110,7 +121,15 @@ def setup(models: List[str] = [], verbose: bool = True, continue_on_fail: bool =
     model_paths = filter(lambda p: True if not models else os.path.basename(p).lower() in models, _list_model_paths())
     for model_path in model_paths:
         print(f"running setup for {model_path}...", end="", flush=True)
+        if test_mode:
+            versions = get_pkg_versions(TORCH_DEPS)
         success, errmsg, stdout_stderr = _install_deps(model_path, verbose=verbose)
+        if test_mode:
+            new_versions = get_pkg_versions(TORCH_DEPS, reload=True)
+            if versions != new_versions:
+                print(f"The torch packages are re-installed after installing the benchmark model {model_path}. \
+                        Before: {versions}, after: {new_versions}")
+                sys.exit(-1)
         if success and errmsg and "No install.py is found" in errmsg:
             print("SKIP - No install.py is found")
         elif success:

torchbenchmark/models/opacus_cifar10/requirements.txt

@@ -1,2 +1,3 @@
+git+https://github.com/pytorch/functorch.git
 # must include the fix https://github.com/pytorch/opacus/pull/426
 opacus>=1.1.2

utils/__init__.py

@@ -7,16 +7,18 @@
                    "required for setup.\n" \
                    "Do you need to use a proxy?"
 
-def get_pkg_versions(packages: List[str]) -> Dict[str, str]:
+def get_pkg_versions(packages: List[str], reload: bool=False) -> Dict[str, str]:
     versions = {}
     for module in packages:
         module = importlib.import_module(module)
-        versions[module] = module.__version__
+        if reload:
+            module = importlib.reload(module)
+        versions[module.__name__] = module.__version__
     return versions
 
 def _test_https(test_url: str = 'https://github.com', timeout: float = 0.5) -> bool:
     try:
         request.urlopen(test_url, timeout=timeout)
     except OSError:
         return False
-    return True
+    return True