Merge pull request #8 from lilt/google-master

Google master

.github/workflows/cmake.yml

@@ -0,0 +1,77 @@
+name: CI for general build
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+      - 'v*'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, ubuntu-18.04, windows-latest, macOS-11 ]
+        arch: [ x64 ]
+        include:
+          - os: windows-latest
+            arch: x86
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+      with:
+        python-version: '3.x'
+        architecture: ${{matrix.arch}}
+
+    - name: Config for Windows
+      if: runner.os == 'Windows'
+      run: |
+        if ("${{matrix.arch}}" -eq "x64") {
+          $msbuildPlatform = "x64"
+        } else {
+          $msbuildPlatform = "Win32"
+        }
+        cmake -A $msbuildPlatform -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+
+    - name: Config for Unix
+      if: runner.os != 'Windows'
+      run: cmake -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+
+    - name: Build
+      run: cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
+
+    - name: Test
+      working-directory: ${{github.workspace}}/build
+      run: ctest -C Release --output-on-failure
+
+    - name: Package
+      working-directory: ${{github.workspace}}/build
+      run: cpack
+
+    - name: Build Python wrapper
+      working-directory: ${{github.workspace}}/python
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+        python setup.py test
+        python setup.py bdist_wheel
+
+    - name: Upload artifcacts
+      uses: actions/upload-artifact@v3
+      with:
+        path: ./build/*.7z
+
+    - name: Upload Release Assets
+      if: startsWith(github.ref, 'refs/tags/')
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: ./build/*.7z
+        tag: ${{ github.ref }}
+        overwrite: true
+        prerelease: true
+        file_glob: true
+        body: "This is my release text"

.github/workflows/wheel.yml

@@ -0,0 +1,136 @@
+name: Build Wheels
+
+on:
+  push:
+    branches: [ master ]
+    tags:
+      - 'v*'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build_wheels:
+    outputs:
+      digests-linux: ${{ steps.hash-linux.outputs.digests }}
+      digests-macos: ${{ steps.hash-macos.outputs.digests }}
+      digests-windows: ${{ steps.hash-windows.outputs.digests }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-11]
+    runs-on: ${{ matrix.os }}
+    name: Build wheels on ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.x"
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
+      - name: Build for Windows
+        if: runner.os == 'Windows'
+        run: |
+          cmake -A Win32 -B ${{github.workspace}}/build_win32 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_win32
+          cmake --build ${{github.workspace}}/build_win32 --config Release --target install --parallel 8
+          cmake -A x64   -B ${{github.workspace}}/build_amd64 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_amd64
+          cmake --build ${{github.workspace}}/build_amd64 --config Release --target install --parallel 8
+
+      - name: Build for Mac
+        if: runner.os == 'macOS'
+        run: |
+          cmake -B ${{github.workspace}}/build -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+          cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
+        env:
+          CMAKE_OSX_ARCHITECTURES: arm64;x86_64
+
+      - name: Install cibuildwheel
+        working-directory: ${{github.workspace}}/python
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+          python -m pip install cibuildwheel==2.6.0
+
+      - name: Build wheels
+        working-directory: ${{github.workspace}}/python
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_ARCHS_LINUX: auto aarch64
+          CIBW_ARCHS_MACOS: x86_64 universal2 arm64
+          CIBW_SKIP: "pp* *-musllinux_*"
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Build sdist
+        working-directory: ${{github.workspace}}/python
+        if: runner.os == 'macOS'
+        run: |
+          python setup.py sdist
+          cp -f dist/*.tar.gz wheelhouse/
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          path: |
+            ./python/wheelhouse/*.whl
+            ./python/dist/*.tar.gz
+
+      - name: Upload wheel release
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./python/wheelhouse/*
+          tag: ${{ github.ref }}
+          overwrite: true
+          prerelease: true
+          file_glob: true
+
+      - name: Generate SLSA subjects - Macos
+        id: hash-macos
+        if: runner.os == 'macOS'
+        run: echo "::set-output name=digests::$(shasum -a 256 ./python/wheelhouse/* | base64)"
+
+      - name: Generate SLSA subjects - Linux
+        id: hash-linux
+        if: runner.os == 'Linux'
+        run: echo "::set-output name=digests::$(sha256sum ./python/wheelhouse/* | base64 -w0)"
+
+      - name: Generate SLSA subjects - Windows
+        id: hash-windows
+        if: runner.os == 'Windows'
+        run: echo "::set-output name=digests::$(sha256sum ./python/wheelhouse/* | base64 -w0)"
+
+  gather-disgests:
+    needs: [build_wheels]
+    outputs:
+      digests: ${{ steps.hash.outputs.digests }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Merge results
+        id: hash
+        env:
+          LINUX_DIGESTS: "${{ needs.build_wheels.outputs.digests-linux }}"
+          MACOS_DIGESTS: "${{ needs.build_wheels.outputs.digests-macos }}"
+          WINDOWS_DIGESTS: "${{ needs.build_wheels.outputs.digests-windows }}"
+        run: |
+          set -euo pipefail
+          echo "$LINUX_DIGESTS" | base64 -d > checksums.txt
+          echo "$MACOS_DIGESTS" | base64 -d >> checksums.txt
+          echo "$WINDOWS_DIGESTS" | base64 -d >> checksums.txt
+          echo "::set-output name=digests::$(cat checksums.txt | base64 -w0)"
+
+  provenance:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: [build_wheels, gather-disgests]
+    permissions:
+      actions: read   # To read the workflow path.
+      id-token: write # To sign the provenance.
+      contents: write # To add assets to a release.
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.2.0
+    with:
+      base64-subjects: "${{ needs.gather-disgests.outputs.digests }}"
+      upload-assets: true # Optional: Upload to a new release

.gitignore

@@ -70,3 +70,6 @@ m.vocab
 cmake_install.cmake
 libsentencepiece.so*
 libsentencepiece_train.so*
+python/bundled
+_sentencepiece.*.so
+third_party/abseil-cpp