[CI] install fix for HF tests and use dict instead of json for some t…

…ests (#1405)

* install HF w. dev extra to get all required packages

* switch ds.init to use param dict instead of json file on disk

* switch back to 'testing' extra

.github/workflows/main.yml

@@ -93,6 +93,8 @@ jobs:
           git clone https://github.com/huggingface/transformers
           git rev-parse --short HEAD
           cd transformers
+          # scipy/sklearn required for tests, using the 'dev' extra forces torch re-install
+          pip install scipy sklearn
           pip install .[testing]
           # find reqs used in ds integration tests
           find examples/pytorch  -regextype posix-egrep -regex '.*(language-modeling|question-answering|summarization|text-classification|translation).*/requirements.txt' -exec pip install -r {} \;

tests/unit/test_configurable_parallel.py

@@ -51,13 +51,11 @@ def get_deepspeed_model(self, model, tmpdir):
             },
         }
 
-        ds_args = args_from_dict(tmpdir, ds_config_dict)
-
         from megatron import mpu
-        model, _, _,_ = deepspeed.initialize(args=ds_args,
-                                            model=model,
-                                            mpu=mpu,
-                                            model_parameters=model.parameters())
+        model, _, _,_ = deepspeed.initialize(model=model,
+                                             mpu=mpu,
+                                             model_parameters=model.parameters(),
+                                             config=ds_config_dict)
         return model
 
     def test_gpt2_basic(self, tmpdir):
@@ -243,13 +241,11 @@ def get_deepspeed_model(self, model, tmpdir):
                 }
             },
         }
-
-        ds_args = args_from_dict(tmpdir, ds_config_dict)
         dist.barrier()
 
-        model, _, _,_ = deepspeed.initialize(args=ds_args,
-                                            model=model,
-                                            model_parameters=model.parameters())
+        model, _, _,_ = deepspeed.initialize(model=model,
+                                             model_parameters=model.parameters(),
+                                             config=ds_config_dict)
         return model.cuda()
 
     def get_topology(self, mp, pp, world_size):