support big endian platform, add powerpc ci (#5121)

Tencent · Nov 1, 2023 · 39bc71c · 39bc71c
1 parent 4494aad
commit 39bc71c
Show file tree

Hide file tree

Showing 7 changed files with 330 additions and 1 deletion.
diff --git a/.ci/test-coverage.yml b/.ci/test-coverage.yml
@@ -596,6 +596,164 @@ jobs:
     - name: codecov
       run: ./codecov -t ${{settings.CODECOV_TOKEN.access_token}} -C ${{ ci.sha }} -B ${{ ci.head_ref }} -f build/lcov.info
 
+  linux-gcc-powerpc:
+    name: linux-gcc-powerpc
+    strategy:
+      matrix:
+        OPENMP: ['OFF', 'ON']
+
+    runs-on:
+      pool-name: docker
+      container:
+        image: bkci/ci:ubuntu
+    steps:
+    - name: checkout
+      checkout: self
+      with:
+        strategy: FRESH_CHECKOUT
+        enableSubmodule: false
+        enableGitLfs: false
+
+    - name: install-deps
+      run: |
+        apt-get update
+        apt-get install -y lcov g++-powerpc-linux-gnu libcapstone4 libglib2.0-0
+        curl https://uploader.codecov.io/verification.gpg | gpg --no-default-keyring --keyring trustedkeys.gpg --import
+        curl -Os https://uploader.codecov.io/latest/linux/codecov
+        curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
+        curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
+        gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
+        shasum -a 256 -c codecov.SHA256SUM
+        chmod +x codecov
+
+    - name: cache-qemu
+      id: cache-qemu
+      uses: cache@1.*
+      with:
+        cachePaths: qemu-install
+        cacheKey: qemu-ppc-install-20220831-1
+
+    - name: checkout-qemu
+      if: steps.cache-qemu.outputs.cacheHit != 'true'
+      checkout: https://github.com/qemu/qemu.git
+      with:
+        pullType: COMMIT_ID
+        refName: 621da7789083b80d6f1ff1c0fb499334007b4f51
+        localPath: qemu
+        enableSubmodule: false
+        enableGitLfs: false
+
+    - name: qemu
+      if: steps.cache-qemu.outputs.cacheHit != 'true'
+      run: |
+        echo 'deb-src http://mirrors.cloud.tencent.com/debian bullseye main' | tee -a /etc/apt/sources.list
+        echo 'deb-src http://mirrors.cloud.tencent.com/debian bullseye-updates main' | tee -a /etc/apt/sources.list
+        apt-get update
+        apt-get build-dep -y qemu
+        cd qemu
+        ./configure --prefix=${{ci.workspace}}/qemu-install --target-list=ppc-linux-user --disable-system
+        make -j$(nproc)
+        make install
+
+    - name: build
+      run: |
+        mkdir build && cd build
+        cmake -DCMAKE_TOOLCHAIN_FILE=../toolchains/powerpc-linux-gnu.toolchain.cmake -DCMAKE_BUILD_TYPE=debug -DNCNN_COVERAGE=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_OPENMP=${{matrix.OPENMP}} -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_TESTS=ON ..
+        cmake --build . -j $(nproc)
+    - name: test
+      run: |
+        export PATH=${{ci.workspace}}/qemu-install/bin:$PATH
+        cd build
+        TESTS_EXECUTABLE_LOADER=qemu-ppc TESTS_EXECUTABLE_LOADER_ARGUMENTS="-L;/usr/powerpc-linux-gnu" ctest --output-on-failure -j $(nproc)
+    - name: lcov-collect
+      run: |
+        cd build
+        lcov -d ./src -c -o lcov.info
+        lcov -r lcov.info '/usr/*' -o lcov.info
+        lcov -r lcov.info '*/build/*' -o lcov.info
+        lcov --list lcov.info
+    - name: codecov
+      run: ./codecov -t ${{settings.CODECOV_TOKEN.access_token}} -C ${{ ci.sha }} -B ${{ ci.head_ref }} -f build/lcov.info
+
+  linux-gcc-powerpc64le:
+    name: linux-gcc-powerpc64le
+    strategy:
+      matrix:
+        OPENMP: ['OFF', 'ON']
+
+    runs-on:
+      pool-name: docker
+      container:
+        image: bkci/ci:ubuntu
+    steps:
+    - name: checkout
+      checkout: self
+      with:
+        strategy: FRESH_CHECKOUT
+        enableSubmodule: false
+        enableGitLfs: false
+
+    - name: install-deps
+      run: |
+        apt-get update
+        apt-get install -y lcov g++-powerpc64le-linux-gnu libcapstone4 libglib2.0-0
+        curl https://uploader.codecov.io/verification.gpg | gpg --no-default-keyring --keyring trustedkeys.gpg --import
+        curl -Os https://uploader.codecov.io/latest/linux/codecov
+        curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
+        curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
+        gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
+        shasum -a 256 -c codecov.SHA256SUM
+        chmod +x codecov
+
+    - name: cache-qemu
+      id: cache-qemu
+      uses: cache@1.*
+      with:
+        cachePaths: qemu-install
+        cacheKey: qemu-ppc64le-install-20220831-1
+
+    - name: checkout-qemu
+      if: steps.cache-qemu.outputs.cacheHit != 'true'
+      checkout: https://github.com/qemu/qemu.git
+      with:
+        pullType: COMMIT_ID
+        refName: 621da7789083b80d6f1ff1c0fb499334007b4f51
+        localPath: qemu
+        enableSubmodule: false
+        enableGitLfs: false
+
+    - name: qemu
+      if: steps.cache-qemu.outputs.cacheHit != 'true'
+      run: |
+        echo 'deb-src http://mirrors.cloud.tencent.com/debian bullseye main' | tee -a /etc/apt/sources.list
+        echo 'deb-src http://mirrors.cloud.tencent.com/debian bullseye-updates main' | tee -a /etc/apt/sources.list
+        apt-get update
+        apt-get build-dep -y qemu
+        cd qemu
+        ./configure --prefix=${{ci.workspace}}/qemu-install --target-list=ppc64le-linux-user --disable-system
+        make -j$(nproc)
+        make install
+
+    - name: build
+      run: |
+        mkdir build && cd build
+        cmake -DCMAKE_TOOLCHAIN_FILE=../toolchains/powerpc64le-linux-gnu.toolchain.cmake -DCMAKE_BUILD_TYPE=debug -DNCNN_COVERAGE=ON -DNCNN_RUNTIME_CPU=OFF -DNCNN_OPENMP=${{matrix.OPENMP}} -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_TESTS=ON ..
+        cmake --build . -j $(nproc)
+    - name: test
+      run: |
+        export PATH=${{ci.workspace}}/qemu-install/bin:$PATH
+        cd build
+        TESTS_EXECUTABLE_LOADER=qemu-ppc64le TESTS_EXECUTABLE_LOADER_ARGUMENTS="-L;/usr/powerpc64le-linux-gnu" ctest --output-on-failure -j $(nproc)
+    - name: lcov-collect
+      run: |
+        cd build
+        lcov -d ./src -c -o lcov.info
+        lcov -r lcov.info '/usr/*' -o lcov.info
+        lcov -r lcov.info '*/build/*' -o lcov.info
+        lcov --list lcov.info
+    - name: codecov
+      run: ./codecov -t ${{settings.CODECOV_TOKEN.access_token}} -C ${{ ci.sha }} -B ${{ ci.head_ref }} -f build/lcov.info
+
   linux-gcc-riscv64:
     name: linux-gcc-riscv64
     strategy:
@@ -951,4 +1109,4 @@ jobs:
         lcov -r lcov.info '*/build/*' -o lcov.info
         lcov --list lcov.info
     - name: codecov
-      run: ./codecov -t ${{settings.CODECOV_TOKEN.access_token}} -C ${{ ci.sha }} -B ${{ ci.head_ref }} -f build/lcov.info
+      run: ./codecov -t ${{settings.CODECOV_TOKEN.access_token}} -C ${{ ci.sha }} -B ${{ ci.head_ref }} -f build/lcov.info
diff --git a/.github/workflows/linux-ppc64-cpu-gcc.yml b/.github/workflows/linux-ppc64-cpu-gcc.yml
@@ -27,6 +27,53 @@ permissions:
   contents: read
 
 jobs:
+  linux-gcc-ppc:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: cache-qemu
+      id: cache-qemu
+      uses: actions/cache@v3
+      with:
+        path: qemu-install
+        key: qemu-ppc-install-20220502-2
+    - name: install-qemu-build-deps
+      if: steps.cache-qemu.outputs.cache-hit != 'true'
+      run: |
+        sudo apt-get update
+        sudo apt-get install autoconf automake autotools-dev ninja-build
+    - name: checkout-qemu
+      if: steps.cache-qemu.outputs.cache-hit != 'true'
+      uses: actions/checkout@v4
+      with:
+        repository: qemu/qemu
+        path: qemu
+        ref: f5643914a9e8f79c606a76e6a9d7ea82a3fc3e65
+    - name: qemu
+      if: steps.cache-qemu.outputs.cache-hit != 'true'
+      run: |
+        cd qemu
+        ./configure --prefix=$GITHUB_WORKSPACE/qemu-install --target-list=ppc-linux-user --disable-system
+        make -j2
+        make install
+
+    - name: powerpc-gnu-toolchain
+      run: |
+        sudo apt-get update
+        sudo apt-get install g++-powerpc-linux-gnu
+
+    - name: configure
+      run: mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=../toolchains/powerpc-linux-gnu.toolchain.cmake -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_TESTS=ON ..
+    - name: build
+      run: cmake --build build -j 2
+
+    - name: test
+      run: |
+        export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH
+        cd build
+        TESTS_EXECUTABLE_LOADER=qemu-ppc TESTS_EXECUTABLE_LOADER_ARGUMENTS="-L;/usr/powerpc-linux-gnu" ctest --output-on-failure -j 2
+
   linux-gcc-ppc64le:
     runs-on: ubuntu-20.04
     steps:
@@ -73,6 +120,7 @@ jobs:
         export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH
         cd build
         TESTS_EXECUTABLE_LOADER=qemu-ppc64le TESTS_EXECUTABLE_LOADER_ARGUMENTS="-L;/usr/powerpc64le-linux-gnu" ctest --output-on-failure -j 2
+
   linux-gcc-power8le-vsx:
     runs-on: ubuntu-20.04
     steps:

diff --git a/src/modelbin.cpp b/src/modelbin.cpp
@@ -117,13 +117,18 @@ Mat ModelBinFromDataReader::load(int w, int type) const
             return Mat();
         }
 
+#if __BIG_ENDIAN__
+        swap_endianness_32(&flag_struct.tag);
+#endif
+
         unsigned int flag = (int)flag_struct.f0 + flag_struct.f1 + flag_struct.f2 + flag_struct.f3;
 
         if (flag_struct.tag == 0x01306B47)
         {
             // half-precision data
             size_t align_data_size = alignSize(w * sizeof(unsigned short), 4);
 
+#if !__BIG_ENDIAN__
             // try reference data
             const void* refbuf = 0;
             nread = d->dr.reference(align_data_size, &refbuf);
@@ -132,6 +137,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                 m = Mat::from_float16((const unsigned short*)refbuf, w);
             }
             else
+#endif
             {
                 std::vector<unsigned short> float16_weights;
                 float16_weights.resize(align_data_size);
@@ -142,6 +148,13 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                     return Mat();
                 }
 
+#if __BIG_ENDIAN__
+                for (int i = 0; i < w; i++)
+                {
+                    swap_endianness_16(&float16_weights[i]);
+                }
+#endif
+
                 m = Mat::from_float16(&float16_weights[0], w);
             }
 
@@ -152,6 +165,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
             // int8 data
             size_t align_data_size = alignSize(w, 4);
 
+#if !__BIG_ENDIAN__
             // try reference data
             const void* refbuf = 0;
             nread = d->dr.reference(align_data_size, &refbuf);
@@ -160,6 +174,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                 m = Mat(w, (void*)refbuf, (size_t)1u);
             }
             else
+#endif
             {
                 std::vector<signed char> int8_weights;
                 int8_weights.resize(align_data_size);
@@ -181,6 +196,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
         }
         else if (flag_struct.tag == 0x0002C056)
         {
+#if !__BIG_ENDIAN__
             // try reference data
             const void* refbuf = 0;
             nread = d->dr.reference(w * sizeof(float), &refbuf);
@@ -189,6 +205,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                 m = Mat(w, (void*)refbuf);
             }
             else
+#endif
             {
                 m.create(w);
                 if (m.empty())
@@ -201,6 +218,13 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                     NCNN_LOGE("ModelBin read weight_data failed %zd", nread);
                     return Mat();
                 }
+
+#if __BIG_ENDIAN__
+                for (int i = 0; i < w; i++)
+                {
+                    swap_endianness_32((float*)m + i);
+                }
+#endif
             }
 
             return m;
@@ -221,6 +245,13 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                 return Mat();
             }
 
+#if __BIG_ENDIAN__
+            for (int i = 0; i < 256; i++)
+            {
+                swap_endianness_32(&quantization_value[i]);
+            }
+#endif
+
             size_t align_weight_data_size = alignSize(w * sizeof(unsigned char), 4);
             std::vector<unsigned char> index_array;
             index_array.resize(align_weight_data_size);
@@ -239,6 +270,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
         }
         else if (flag_struct.f0 == 0)
         {
+#if !__BIG_ENDIAN__
             // try reference data
             const void* refbuf = 0;
             nread = d->dr.reference(w * sizeof(float), &refbuf);
@@ -247,6 +279,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                 m = Mat(w, (void*)refbuf);
             }
             else
+#endif
             {
                 m.create(w);
                 if (m.empty())
@@ -259,13 +292,21 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                     NCNN_LOGE("ModelBin read weight_data failed %zd", nread);
                     return Mat();
                 }
+
+#if __BIG_ENDIAN__
+                for (int i = 0; i < w; i++)
+                {
+                    swap_endianness_32((float*)m + i);
+                }
+#endif
             }
         }
 
         return m;
     }
     else if (type == 1)
     {
+#if !__BIG_ENDIAN__
         // try reference data
         const void* refbuf = 0;
         size_t nread = d->dr.reference(w * sizeof(float), &refbuf);
@@ -274,6 +315,7 @@ Mat ModelBinFromDataReader::load(int w, int type) const
             m = Mat(w, (void*)refbuf);
         }
         else
+#endif
         {
             m.create(w);
             if (m.empty())
@@ -286,6 +328,13 @@ Mat ModelBinFromDataReader::load(int w, int type) const
                 NCNN_LOGE("ModelBin read weight_data failed %zd", nread);
                 return Mat();
             }
+
+#if __BIG_ENDIAN__
+            for (int i = 0; i < w; i++)
+            {
+                swap_endianness_32((float*)m + i);
+            }
+#endif
         }
 
         return m;