diff --git a/.circleci/config.yml b/.circleci/config.yml index 031895e8f0..da15e308d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: command: ./scripts/install_nightlies.sh - run: name: Setup benchmark suite dependencies - command: . ~/miniconda3/etc/profile.d/conda.sh; conda activate base; python install.py + command: . ~/miniconda3/etc/profile.d/conda.sh; conda activate base; python install.py --test-mode - run: name: Validate benchmark components command: | diff --git a/install.py b/install.py index 9f6b72c3b6..7464017d01 100644 --- a/install.py +++ b/install.py @@ -77,6 +77,7 @@ def pip_install_requirements(requirements_txt="requirements.txt"): parser = argparse.ArgumentParser() parser.add_argument("models", nargs='*', default=[], help="Specify one or more models to install. If not set, install all models.") + parser.add_argument("--test-mode", action="store_true", help="Run in test mode and check package versions") parser.add_argument("--continue_on_fail", action="store_true") parser.add_argument("--verbose", "-v", action="store_true") parser.add_argument("--component", choices=["distributed"], help="Install requirements for optional components.") @@ -125,7 +126,7 @@ def pip_install_requirements(requirements_txt="requirements.txt"): Before: {versions}, after: {new_versions}") sys.exit(-1) from torchbenchmark import setup - success &= setup(models=args.models, verbose=args.verbose, continue_on_fail=args.continue_on_fail) + success &= setup(models=args.models, verbose=args.verbose, continue_on_fail=args.continue_on_fail, test_mode=args.test_mode) if not success: if args.continue_on_fail: print("Warning: some benchmarks were not installed due to failure") diff --git a/submodules/FAMBench b/submodules/FAMBench index a0f12ca4fe..4f815ecc17 160000 --- a/submodules/FAMBench +++ b/submodules/FAMBench @@ -1 +1 @@ -Subproject commit a0f12ca4fe8973f4cc65d18b51ce3aa94ceec0ac +Subproject commit 4f815ecc17b6e937d62834ad0f846ac4dfc9d844 diff --git a/torchbenchmark/__init__.py b/torchbenchmark/__init__.py index c7f823fd07..88ae1ea59f 100644 --- a/torchbenchmark/__init__.py +++ b/torchbenchmark/__init__.py @@ -19,10 +19,21 @@ REPO_PATH = Path(os.path.abspath(__file__)).parent.parent DATA_PATH = os.path.join(REPO_PATH, "torchbenchmark", "data", ".data") -TORCH_DEPS = ['torch', 'torchvision', 'torchtext'] -proxy_suggestion = "Unable to verify https connectivity, " \ - "required for setup.\n" \ - "Do you need to use a proxy?" +class add_path(): + def __init__(self, path): + self.path = path + + def __enter__(self): + sys.path.insert(0, self.path) + + def __exit__(self, exc_type, exc_value, traceback): + try: + sys.path.remove(self.path) + except ValueError: + pass + +with add_path(str(REPO_PATH)): + from utils import TORCH_DEPS, get_pkg_versions, proxy_suggestion this_dir = pathlib.Path(__file__).parent.absolute() model_dir = 'models' @@ -100,7 +111,7 @@ def _is_internal_model(model_name: str) -> bool: return True return False -def setup(models: List[str] = [], verbose: bool = True, continue_on_fail: bool = False) -> bool: +def setup(models: List[str] = [], verbose: bool = True, continue_on_fail: bool = False, test_mode: bool = False) -> bool: if not _test_https(): print(proxy_suggestion) sys.exit(-1) @@ -110,7 +121,15 @@ def setup(models: List[str] = [], verbose: bool = True, continue_on_fail: bool = model_paths = filter(lambda p: True if not models else os.path.basename(p).lower() in models, _list_model_paths()) for model_path in model_paths: print(f"running setup for {model_path}...", end="", flush=True) + if test_mode: + versions = get_pkg_versions(TORCH_DEPS) success, errmsg, stdout_stderr = _install_deps(model_path, verbose=verbose) + if test_mode: + new_versions = get_pkg_versions(TORCH_DEPS, reload=True) + if versions != new_versions: + print(f"The torch packages are re-installed after installing the benchmark model {model_path}. \ + Before: {versions}, after: {new_versions}") + sys.exit(-1) if success and errmsg and "No install.py is found" in errmsg: print("SKIP - No install.py is found") elif success: diff --git a/torchbenchmark/models/opacus_cifar10/requirements.txt b/torchbenchmark/models/opacus_cifar10/requirements.txt index cdba2625a1..106f0456b9 100644 --- a/torchbenchmark/models/opacus_cifar10/requirements.txt +++ b/torchbenchmark/models/opacus_cifar10/requirements.txt @@ -1,2 +1,3 @@ +git+https://github.com/pytorch/functorch.git # must include the fix https://github.com/pytorch/opacus/pull/426 opacus>=1.1.2 diff --git a/utils/__init__.py b/utils/__init__.py index c5024d7a62..1f02075b18 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -7,11 +7,13 @@ "required for setup.\n" \ "Do you need to use a proxy?" -def get_pkg_versions(packages: List[str]) -> Dict[str, str]: +def get_pkg_versions(packages: List[str], reload: bool=False) -> Dict[str, str]: versions = {} for module in packages: module = importlib.import_module(module) - versions[module] = module.__version__ + if reload: + module = importlib.reload(module) + versions[module.__name__] = module.__version__ return versions def _test_https(test_url: str = 'https://github.com', timeout: float = 0.5) -> bool: @@ -19,4 +21,4 @@ def _test_https(test_url: str = 'https://github.com', timeout: float = 0.5) -> b request.urlopen(test_url, timeout=timeout) except OSError: return False - return True \ No newline at end of file + return True