Merge branch 'master' into fix_z3

microsoft · Nov 12, 2024 · 12189fe · 12189fe
2 parents 996f88f + 7af3a4b
commit 12189fe
Show file tree

Hide file tree

Showing 24 changed files with 869 additions and 100 deletions.
diff --git a/.github/workflows/amd-mi200.yml b/.github/workflows/amd-mi200.yml
@@ -48,8 +48,6 @@ jobs:
       - name: Install (ROCm) apex
         run: |
           git clone https://github.com/ROCmSoftwarePlatform/apex.git
-          cd apex
-          git checkout torch_2.1_higher
           CURRENT_VER=$(git rev-parse HEAD)
           INSTALLED_VER=$(cat /blob/amd-apex/.venv_installed_version)
           if [[ "$CURRENT_VER" != "$INSTALLED_VER" ]]; then

diff --git a/.github/workflows/cpu-torch-latest.yml b/.github/workflows/cpu-torch-latest.yml
@@ -19,7 +19,7 @@ concurrency:
 
 jobs:
   unit-tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/no-torch.yml b/.github/workflows/no-torch.yml
@@ -19,7 +19,7 @@ permissions:
 
 jobs:
   unit-tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: actions/checkout@v4
@@ -30,6 +30,7 @@ jobs:
       - name: Python environment
         run: |
           pip uninstall torch --yes
+          pip install setuptools
           pip list
 
       - name: Build deepspeed

diff --git a/.github/workflows/nv-pre-compile-ops.yml b/.github/workflows/nv-pre-compile-ops.yml
@@ -21,7 +21,7 @@ concurrency:
 
 jobs:
   unit-tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     container:
       image: deepspeed/gh-builder:ubuntu1804-py38-torch1131-cu116
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   deploy:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     environment: release-env
 
     steps:

diff --git a/COMMITTERS.md b/COMMITTERS.md
@@ -0,0 +1,9 @@
+# DeepSpeed TSC Committers #
+
+| Name | GitHub ID | Affiliation
+|--- | ---- | --- |
+| Olatunji Ruwase | [tjruwase](https://github.com/tjruwase)     | Microsoft |
+| Logan Adams     | [loadams](https://github.com/loadams)      | Microsoft |
+| Masahiro Tanaka | [tohtana](https://github.com/tohtana)      | Microsoft |
+| Jeff Rasley     | [jeffra](https://github.com/jeffra)       | SnowFlake |
+| Minjia Zhang    | [minjiazhang](https://github.com/minjiazhang)  | UIUC      |
diff --git a/csrc/adagrad/cpu_adagrad.cpp b/csrc/adagrad/cpu_adagrad.cpp
@@ -17,9 +17,9 @@ static std::unordered_map<int, std::shared_ptr<void>> s_optimizers;
 
 // C++ interface
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
-void Adagrad_Optimizer::Step_1(ds_params_percision_t* _params,
-                               ds_params_percision_t* grads,
+template <typename ds_params_precision_t, typename ds_state_precision_t>
+void Adagrad_Optimizer::Step_1(ds_params_precision_t* _params,
+                               ds_params_precision_t* grads,
                                ds_state_precision_t* _exp_avg_sq,
                                size_t _param_size)
 {
@@ -56,9 +56,9 @@ void Adagrad_Optimizer::Step_1(ds_params_percision_t* _params,
     }
 }
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
-void Adagrad_Optimizer::Step_4(ds_params_percision_t* _params,
-                               ds_params_percision_t* grads,
+template <typename ds_params_precision_t, typename ds_state_precision_t>
+void Adagrad_Optimizer::Step_4(ds_params_precision_t* _params,
+                               ds_params_precision_t* grads,
                                ds_state_precision_t* _exp_avg_sq,
                                size_t _param_size)
 {
@@ -104,9 +104,9 @@ int create_adagrad_optimizer(int optimizer_id,
     return 0;
 }
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
-void Adagrad_Optimizer::Step_8(ds_params_percision_t* _params,
-                               ds_params_percision_t* grads,
+template <typename ds_params_precision_t, typename ds_state_precision_t>
+void Adagrad_Optimizer::Step_8(ds_params_precision_t* _params,
+                               ds_params_precision_t* grads,
                                ds_state_precision_t* _exp_avg_sq,
                                size_t _param_size)
 {
@@ -121,15 +121,15 @@ void Adagrad_Optimizer::Step_8(ds_params_percision_t* _params,
                (_param_size - rounded_size));
 }
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
+template <typename ds_params_precision_t, typename ds_state_precision_t>
 void step_invoker(std::shared_ptr<Adagrad_Optimizer> opt,
                   void* _params,
                   void* grads,
                   void* _exp_avg_sq,
                   size_t _param_size)
 {
-    opt->Step_8((ds_params_percision_t*)(_params),
-                (ds_params_percision_t*)(grads),
+    opt->Step_8((ds_params_precision_t*)(_params),
+                (ds_params_precision_t*)(grads),
                 (ds_state_precision_t*)(_exp_avg_sq),
                 _param_size);
 }
@@ -139,12 +139,12 @@ std::map<std::tuple<c10::ScalarType, c10::ScalarType>,
     invokers;
 
 // Fill map with template functions for each type
-template <class ds_params_percision_t, class ds_state_precision_t>
+template <class ds_params_precision_t, class ds_state_precision_t>
 void create_invoker()
 {
-    invokers[std::tuple(c10::CppTypeToScalarType<ds_params_percision_t>(),
+    invokers[std::tuple(c10::CppTypeToScalarType<ds_params_precision_t>(),
                         c10::CppTypeToScalarType<ds_state_precision_t>())] =
-        step_invoker<ds_params_percision_t, ds_state_precision_t>;
+        step_invoker<ds_params_precision_t, ds_state_precision_t>;
 }
 struct InvokerInitializer {
     InvokerInitializer()

diff --git a/csrc/adam/cpu_adam_impl.cpp b/csrc/adam/cpu_adam_impl.cpp
@@ -18,9 +18,9 @@ static std::unordered_map<int, std::shared_ptr<void>> s_optimizers;
 
 // C++ interface
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
-void Adam_Optimizer::Step_1(ds_params_percision_t* _params,
-                            ds_params_percision_t* grads,
+template <typename ds_params_precision_t, typename ds_state_precision_t>
+void Adam_Optimizer::Step_1(ds_params_precision_t* _params,
+                            ds_params_precision_t* grads,
                             ds_state_precision_t* _exp_avg,
                             ds_state_precision_t* _exp_avg_sq,
                             size_t _param_size)
@@ -67,9 +67,9 @@ void Adam_Optimizer::Step_1(ds_params_percision_t* _params,
     }
 }
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
-void Adam_Optimizer::Step_4(ds_params_percision_t* _params,
-                            ds_params_percision_t* grads,
+template <typename ds_params_precision_t, typename ds_state_precision_t>
+void Adam_Optimizer::Step_4(ds_params_precision_t* _params,
+                            ds_params_precision_t* grads,
                             ds_state_precision_t* _exp_avg,
                             ds_state_precision_t* _exp_avg_sq,
                             size_t _param_size)
@@ -126,9 +126,9 @@ int create_adam_optimizer(int optimizer_id,
     return 0;
 }
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
-void Adam_Optimizer::Step_8(ds_params_percision_t* _params,
-                            ds_params_percision_t* grads,
+template <typename ds_params_precision_t, typename ds_state_precision_t>
+void Adam_Optimizer::Step_8(ds_params_precision_t* _params,
+                            ds_params_precision_t* grads,
                             ds_state_precision_t* _exp_avg,
                             ds_state_precision_t* _exp_avg_sq,
                             size_t _param_size)
@@ -145,16 +145,16 @@ void Adam_Optimizer::Step_8(ds_params_percision_t* _params,
                (_param_size - rounded_size));
 }
 
-template <typename ds_params_percision_t, typename ds_state_precision_t>
+template <typename ds_params_precision_t, typename ds_state_precision_t>
 void step_invoker(std::shared_ptr<Adam_Optimizer> opt,
                   void* _params,
                   void* grads,
                   void* _exp_avg,
                   void* _exp_avg_sq,
                   size_t _param_size)
 {
-    opt->Step_8((ds_params_percision_t*)(_params),
-                (ds_params_percision_t*)(grads),
+    opt->Step_8((ds_params_precision_t*)(_params),
+                (ds_params_precision_t*)(grads),
                 (ds_state_precision_t*)(_exp_avg),
                 (ds_state_precision_t*)(_exp_avg_sq),
                 _param_size);
@@ -165,12 +165,12 @@ std::map<std::tuple<c10::ScalarType, c10::ScalarType>,
     invokers;
 
 // Fill map with template functions for each type
-template <class ds_params_percision_t, class ds_state_precision_t>
+template <class ds_params_precision_t, class ds_state_precision_t>
 void create_invoker()
 {
-    invokers[std::tuple(c10::CppTypeToScalarType<ds_params_percision_t>(),
+    invokers[std::tuple(c10::CppTypeToScalarType<ds_params_precision_t>(),
                         c10::CppTypeToScalarType<ds_state_precision_t>())] =
-        step_invoker<ds_params_percision_t, ds_state_precision_t>;
+        step_invoker<ds_params_precision_t, ds_state_precision_t>;
 }
 struct InvokerInitializer {
     InvokerInitializer()

diff --git a/csrc/includes/cpu_adagrad.h b/csrc/includes/cpu_adagrad.h
@@ -14,9 +14,9 @@
 #include "simd.h"
 
 #define STEP(SPAN)                                                           \
-    template <typename ds_params_percision_t, typename ds_state_precision_t> \
-    void Step_##SPAN(ds_params_percision_t* _params,                         \
-                     ds_params_percision_t* grads,                           \
+    template <typename ds_params_precision_t, typename ds_state_precision_t> \
+    void Step_##SPAN(ds_params_precision_t* _params,                         \
+                     ds_params_precision_t* grads,                           \
                      ds_state_precision_t* _exp_avg_sq,                      \
                      size_t _param_size);
 
@@ -28,10 +28,10 @@ class Adagrad_Optimizer {
     }
     ~Adagrad_Optimizer() {}
 #if defined(__AVX512__) or defined(__AVX256__)
-    template <int span, typename ds_params_percision_t, typename ds_state_precision_t>
+    template <int span, typename ds_params_precision_t, typename ds_state_precision_t>
     void Step_AVX(size_t* rounded_size,
-                  ds_params_percision_t* _params,
-                  ds_params_percision_t* grads,
+                  ds_params_precision_t* _params,
+                  ds_params_precision_t* grads,
                   ds_state_precision_t* _exp_avg_sq,
                   size_t param_size);
 #endif
@@ -61,15 +61,15 @@ class Adagrad_Optimizer {
 };
 
 #if defined(__AVX512__) or defined(__AVX256__)
-template <int span, typename ds_params_percision_t, typename ds_state_precision_t>
+template <int span, typename ds_params_precision_t, typename ds_state_precision_t>
 void Adagrad_Optimizer::Step_AVX(size_t* rounded_size,
-                                 ds_params_percision_t* _params,
-                                 ds_params_percision_t* grads,
+                                 ds_params_precision_t* _params,
+                                 ds_params_precision_t* grads,
                                  ds_state_precision_t* _exp_avg_sq,
                                  size_t _param_size)
 {
 #if !defined(__AVX512__)
-    if (std::is_same_v<ds_params_percision_t, c10::BFloat16> ||
+    if (std::is_same_v<ds_params_precision_t, c10::BFloat16> ||
         std::is_same_v<ds_state_precision_t, c10::BFloat16>) {
         return;
     }

diff --git a/csrc/includes/cpu_adam.h b/csrc/includes/cpu_adam.h
@@ -14,9 +14,9 @@
 #include "simd.h"
 
 #define STEP(SPAN)                                                           \
-    template <typename ds_params_percision_t, typename ds_state_precision_t> \
-    void Step_##SPAN(ds_params_percision_t* _params,                         \
-                     ds_params_percision_t* grads,                           \
+    template <typename ds_params_precision_t, typename ds_state_precision_t> \
+    void Step_##SPAN(ds_params_precision_t* _params,                         \
+                     ds_params_precision_t* grads,                           \
                      ds_state_precision_t* _exp_avg,                         \
                      ds_state_precision_t* _exp_avg_sq,                      \
                      size_t _param_size);
@@ -43,10 +43,10 @@ class Adam_Optimizer {
     ~Adam_Optimizer() {}
 
 #if defined(__AVX512__) or defined(__AVX256__)
-    template <int span, typename ds_params_percision_t, typename ds_state_precision_t>
+    template <int span, typename ds_params_precision_t, typename ds_state_precision_t>
     void Step_AVX(size_t* rounded_size,
-                  ds_params_percision_t* _params,
-                  ds_params_percision_t* grads,
+                  ds_params_precision_t* _params,
+                  ds_params_precision_t* grads,
                   ds_state_precision_t* _exp_avg,
                   ds_state_precision_t* _exp_avg_sq,
                   size_t param_size);
@@ -106,16 +106,16 @@ class Adam_Optimizer {
 };
 
 #if defined(__AVX512__) or defined(__AVX256__)
-template <int span, typename ds_params_percision_t, typename ds_state_precision_t>
+template <int span, typename ds_params_precision_t, typename ds_state_precision_t>
 void Adam_Optimizer::Step_AVX(size_t* rounded_size,
-                              ds_params_percision_t* _params,
-                              ds_params_percision_t* grads,
+                              ds_params_precision_t* _params,
+                              ds_params_precision_t* grads,
                               ds_state_precision_t* _exp_avg,
                               ds_state_precision_t* _exp_avg_sq,
                               size_t _param_size)
 {
 #if !defined(__AVX512__)
-    if (std::is_same_v<ds_params_percision_t, c10::BFloat16> ||
+    if (std::is_same_v<ds_params_precision_t, c10::BFloat16> ||
         std::is_same_v<ds_state_precision_t, c10::BFloat16>) {
         return;
     }

diff --git a/csrc/includes/cpu_lion.h b/csrc/includes/cpu_lion.h
@@ -14,9 +14,9 @@
 #include "simd.h"
 
 #define STEP(SPAN)                                                           \
-    template <typename ds_params_percision_t, typename ds_state_precision_t> \
-    void Step_##SPAN(ds_params_percision_t* _params,                         \
-                     ds_params_percision_t* grads,                           \
+    template <typename ds_params_precision_t, typename ds_state_precision_t> \
+    void Step_##SPAN(ds_params_precision_t* _params,                         \
+                     ds_params_precision_t* grads,                           \
                      ds_state_precision_t* _exp_avg,                         \
                      size_t _param_size);
 
@@ -32,10 +32,10 @@ class Lion_Optimizer {
     ~Lion_Optimizer() {}
 
 #if defined(__AVX512__) or defined(__AVX256__)
-    template <int span, typename ds_params_percision_t, typename ds_state_precision_t>
+    template <int span, typename ds_params_precision_t, typename ds_state_precision_t>
     void Step_AVX(size_t* rounded_size,
-                  ds_params_percision_t* _params,
-                  ds_params_percision_t* grads,
+                  ds_params_precision_t* _params,
+                  ds_params_precision_t* grads,
                   ds_state_precision_t* _exp_avg,
                   size_t param_size);
 #endif
@@ -67,15 +67,15 @@ class Lion_Optimizer {
 };
 
 #if defined(__AVX512__) or defined(__AVX256__)
-template <int span, typename ds_params_percision_t, typename ds_state_precision_t>
+template <int span, typename ds_params_precision_t, typename ds_state_precision_t>
 void Lion_Optimizer::Step_AVX(size_t* rounded_size,
-                              ds_params_percision_t* _params,
-                              ds_params_percision_t* grads,
+                              ds_params_precision_t* _params,
+                              ds_params_precision_t* grads,
                               ds_state_precision_t* _exp_avg,
                               size_t _param_size)
 {
 #if !defined(__AVX512__)
-    if (std::is_same_v<ds_params_percision_t, c10::BFloat16> ||
+    if (std::is_same_v<ds_params_precision_t, c10::BFloat16> ||
         std::is_same_v<ds_state_precision_t, c10::BFloat16>) {
         return;
     }