fix layernorm-param issue & compilation on newer version of torch

deepspeed/__init__.py

@@ -234,7 +234,8 @@ def init_inference(model,
                    ep_size=1,
                    moe=False,
                    moe_experts=1,
-                   moe_type='standard'):
+                   moe_type='standard',
+                   args=None):
     """Initialize the DeepSpeed InferenceEngine.
 
     Arguments:
@@ -298,6 +299,7 @@ def init_inference(model,
                              replace_with_kernel_inject,
                              moe,
                              moe_experts,
-                             moe_type)
+                             moe_type,
+                             args)
 
     return engine

deepspeed/inference/engine.py

@@ -43,7 +43,8 @@ def __init__(self,
                  replace_with_kernel_inject=False,
                  moe=False,
                  moe_experts=1,
-                 moe_type='standard'):
+                 moe_type='standard',
+                 config=None):
         """
         Args:
             model: torch.nn.Module
@@ -68,7 +69,7 @@ def __init__(self,
 
         self.module = model
 
-        self._get_model_config_generate()
+        self._get_model_config_generate(config)
 
         self.mp_world_size = mp_size
         self.checkpoint = checkpoint
@@ -133,8 +134,8 @@ def __init__(self,
         else:
             self.module.register_forward_pre_hook(self._pre_forward_hook)
 
-    def _get_model_config_generate(self):
-        self.config = getattr(self.module, 'config', None)
+    def _get_model_config_generate(self, config):
+        self.config = getattr(self.module, 'config', None) if config is None else config
         self.generate = getattr(self.module, 'generate', None)
 
     def _create_model_parallel_group(self):

deepspeed/module_inject/replace_module.py

@@ -271,9 +271,12 @@ def replace_with_policy(child,
                     heads=num_attention_heads,
                     layer_norm_eps=config.layer_norm_eps if hasattr(
                         config,
-                        'layer_norm_eps') else (config.layer_norm_epsilon if hasattr(
-                            config,
-                            'layer_norm_epsilon') else 1.0e-5),
+                        'layer_norm_eps') else
+                    (config.layer_norm_epsilon
+                     if hasattr(config,
+                                'layer_norm_epsilon') else config.layernorm_epsilon
+                     if hasattr(config,
+                                'layernorm_epsilon') else 1.0e-12),
                     fp16=fp16,
                     pre_layer_norm=preln,
                     mp_size=mp_size,

op_builder/stochastic_transformer.py

@@ -14,6 +14,9 @@ def __init__(self):
     def absolute_name(self):
         return f'deepspeed.ops.transformer.{self.NAME}_op'
 
+    def extra_ldflags(self):
+        return ['-lcurand']
+
     def nvcc_args(self):
         args = super().nvcc_args()
         args.append('-D__STOCHASTIC_MODE__')

op_builder/transformer.py

@@ -15,6 +15,9 @@ def __init__(self, name=None):
     def absolute_name(self):
         return f'deepspeed.ops.transformer.{self.NAME}_op'
 
+    def extra_ldflags(self):
+        return ['-lcurand']
+
     def sources(self):
         return [
             'csrc/transformer/ds_transformer_cuda.cpp',

op_builder/transformer_inference.py

@@ -22,5 +22,8 @@ def sources(self):
             'csrc/transformer/inference/csrc/apply_rotary_pos_emb.cu',
         ]
 
+    def extra_ldflags(self):
+        return ['-lcurand']
+
     def include_paths(self):
         return ['csrc/transformer/inference/includes']