From 9e5c0c5c3ecabb68b7e9dffac0e9b8d167e3cab8 Mon Sep 17 00:00:00 2001
From: Jeff Rasley <jerasley@microsoft.com>
Date: Mon, 27 Sep 2021 17:34:49 -0700
Subject: [PATCH] [CI] install fix for HF tests and use dict instead of json
 for some tests (#1405)

* install HF w. dev extra to get all required packages

* switch ds.init to use param dict instead of json file on disk

* switch back to 'testing' extra
---
 .github/workflows/main.yml               |  2 ++
 tests/unit/test_configurable_parallel.py | 18 +++++++-----------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 69e3da34e05f..818a28523cda 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -93,6 +93,8 @@ jobs:
           git clone https://github.com/huggingface/transformers
           git rev-parse --short HEAD
           cd transformers
+          # scipy/sklearn required for tests, using the 'dev' extra forces torch re-install
+          pip install scipy sklearn
           pip install .[testing]
           # find reqs used in ds integration tests
           find examples/pytorch  -regextype posix-egrep -regex '.*(language-modeling|question-answering|summarization|text-classification|translation).*/requirements.txt' -exec pip install -r {} \;
diff --git a/tests/unit/test_configurable_parallel.py b/tests/unit/test_configurable_parallel.py
index 322a654a7b4e..b1c4697a16bb 100755
--- a/tests/unit/test_configurable_parallel.py
+++ b/tests/unit/test_configurable_parallel.py
@@ -51,13 +51,11 @@ def get_deepspeed_model(self, model, tmpdir):
             },
         }
 
-        ds_args = args_from_dict(tmpdir, ds_config_dict)
-
         from megatron import mpu
-        model, _, _,_ = deepspeed.initialize(args=ds_args,
-                                            model=model,
-                                            mpu=mpu,
-                                            model_parameters=model.parameters())
+        model, _, _,_ = deepspeed.initialize(model=model,
+                                             mpu=mpu,
+                                             model_parameters=model.parameters(),
+                                             config=ds_config_dict)
         return model
 
     def test_gpt2_basic(self, tmpdir):
@@ -243,13 +241,11 @@ def get_deepspeed_model(self, model, tmpdir):
                 }
             },
         }
-
-        ds_args = args_from_dict(tmpdir, ds_config_dict)
         dist.barrier()
 
-        model, _, _,_ = deepspeed.initialize(args=ds_args,
-                                            model=model,
-                                            model_parameters=model.parameters())
+        model, _, _,_ = deepspeed.initialize(model=model,
+                                             model_parameters=model.parameters(),
+                                             config=ds_config_dict)
         return model.cuda()
 
     def get_topology(self, mp, pp, world_size):