ビルドシステムをPyInstallerへ移行する (#504)

.gitignore

@@ -36,12 +36,6 @@ share/python-wheels/
 *.egg
 MANIFEST
 
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt

Dockerfile

@@ -185,9 +185,6 @@ COPY --from=compile-python-env /opt/python /opt/python
 ADD ./requirements.txt /tmp/
 RUN <<EOF
     # Install requirements
-    # FIXME: Nuitka cannot build with setuptools>=60.7.0
-    # https://github.com/Nuitka/Nuitka/issues/1406
-    gosu user /opt/python/bin/python3 -m pip install --upgrade pip setuptools==60.6.0 wheel
     gosu user /opt/python/bin/pip3 install -r /tmp/requirements.txt
 EOF
 
@@ -272,125 +269,3 @@ CMD [ "gosu", "user", "/opt/python/bin/python3", "./run.py", "--voicelib_dir", "
 # Enable use_gpu
 FROM runtime-env AS runtime-nvidia-env
 CMD [ "gosu", "user", "/opt/python/bin/python3", "./run.py", "--use_gpu", "--voicelib_dir", "/opt/voicevox_core/", "--runtime_dir", "/opt/onnxruntime/lib", "--host", "0.0.0.0" ]
-
-# Binary build environment (common to CPU, GPU)
-FROM runtime-env AS build-env
-
-# Install ccache for Nuitka cache
-# chrpath: required for nuitka build; 'RPATH' settings in used shared
-RUN <<EOF
-    set -eux
-
-    apt-get update
-    apt-get install -y \
-        ccache \
-        chrpath \
-        patchelf
-    apt-get clean
-    rm -rf /var/lib/apt/lists/*
-EOF
-
-# Install Python build dependencies
-ADD ./requirements-dev.txt /tmp/
-RUN <<EOF
-    set -eux
-
-    gosu user /opt/python/bin/pip3 install -r /tmp/requirements-dev.txt
-EOF
-
-# Generate licenses.json with build dependencies
-RUN <<EOF
-    set -eux
-
-    cd /opt/voicevox_engine
-
-    # Define temporary env vars
-    # /home/user/.local/bin is required to use the commands installed by pip
-    export PATH="/home/user/.local/bin:${PATH:-}"
-
-    gosu user /opt/python/bin/pip3 install pip-licenses
-    gosu user /opt/python/bin/python3 generate_licenses.py > /opt/voicevox_engine/engine_manifest_assets/dependency_licenses.json
-    # FIXME: VOICEVOX (editor) cannot build without licenses.json
-    cp /opt/voicevox_engine/engine_manifest_assets/dependency_licenses.json /opt/voicevox_engine/licenses.json
-EOF
-
-# Create build script
-RUN <<EOF
-    set -eux
-
-    cat <<EOD > /build.sh
-        #!/bin/bash
-        set -eux
-
-        # chown general user c.z. mounted directory may be owned by root
-        mkdir -p /opt/voicevox_engine_build
-        chown -R user:user /opt/voicevox_engine_build
-
-        mkdir -p /home/user/.cache/Nuitka
-        chown -R user:user /home/user/.cache/Nuitka
-
-        cd /opt/voicevox_engine_build
-
-        gosu user /opt/python/bin/python3 -m nuitka \
-            --output-dir=/opt/voicevox_engine_build \
-            --standalone \
-            --plugin-enable=numpy \
-            --plugin-enable=multiprocessing \
-            --follow-import-to=numpy \
-            --follow-import-to=aiofiles \
-            --include-package=uvicorn \
-            --include-package=anyio \
-            --include-package-data=pyopenjtalk \
-            --include-package-data=scipy \
-            --include-data-file=/opt/voicevox_engine/licenses.json=./ \
-            --include-data-file=/opt/voicevox_engine/presets.yaml=./ \
-            --include-data-file=/opt/voicevox_engine/default.csv=./ \
-            --include-data-file=/opt/voicevox_engine/engine_manifest.json=./ \
-            --include-data-file=/opt/voicevox_core/*.so=./ \
-            --include-data-file=/opt/onnxruntime/lib/libonnxruntime.so=./ \
-            --include-data-dir=/opt/voicevox_engine/speaker_info=./speaker_info \
-            --include-data-dir=/opt/voicevox_engine/engine_manifest_assets=./engine_manifest_assets \
-            --follow-imports \
-            --no-prefer-source-code \
-            /opt/voicevox_engine/run.py
-
-        # Copy libonnxruntime_providers_cuda.so and dependencies (CUDA/cuDNN)
-        if [ -f "/opt/onnxruntime/lib/libonnxruntime_providers_cuda.so" ]; then
-            mkdir -p /tmp/coredeps
-
-            # Copy provider libraries (libonnxruntime.so.{version} is copied by Nuitka)
-            cd /opt/onnxruntime/lib
-            cp libonnxruntime_*.so /tmp/coredeps/
-            cd -
-
-            # assert nvidia/cuda base image
-            cd /usr/local/cuda/lib64
-            cp libcublas.so.* libcublasLt.so.* libcudart.so.* libcufft.so.* libcurand.so.* /tmp/coredeps/
-            cd -
-
-            # remove unneed full version libraries
-            cd /tmp/coredeps
-            rm -f libcublas.so.*.* libcublasLt.so.*.* libcufft.so.*.* libcurand.so.*.*
-            rm -f libcudart.so.*.*.*
-            cd -
-
-            # assert nvidia/cuda base image
-            cd /usr/lib/x86_64-linux-gnu
-            cp libcudnn.so.* libcudnn_*_infer.so.* /tmp/coredeps/
-            cd -
-
-            # remove unneed full version libraries
-            cd /tmp/coredeps
-            rm -f libcudnn.so.*.* libcudnn_*_infer.so.*.*
-            cd -
-
-            mv /tmp/coredeps/* /opt/voicevox_engine_build/run.dist/
-            rm -rf /tmp/coredeps
-        fi
-
-        chmod +x /opt/voicevox_engine_build/run.dist/run
-EOD
-    chmod +x /build.sh
-EOF
-
-CMD [ "bash", "/build.sh" ]

Makefile

@@ -140,7 +140,6 @@ build-linux-docker-build-ubuntu20.04:
 .PHONY: run-linux-docker-build-ubuntu20.04
 run-linux-docker-build-ubuntu20.04:
 	docker run --rm -it \
-		-v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \
 		-v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \
 		voicevox/voicevox_engine:build-cpu-ubuntu20.04-latest $(CMD)
 
@@ -158,7 +157,6 @@ build-linux-docker-build-nvidia-ubuntu20.04:
 .PHONY: run-linux-docker-build-nvidia-ubuntu20.04
 run-linux-docker-build-nvidia-ubuntu20.04:
 	docker run --rm -it \
-		-v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \
 		-v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \
 		voicevox/voicevox_engine:build-nvidia-ubuntu20.04-latest $(CMD)
 
@@ -177,7 +175,6 @@ build-linux-docker-build-ubuntu18.04:
 .PHONY: run-linux-docker-build-ubuntu18.04
 run-linux-docker-build-ubuntu18.04:
 	docker run --rm -it \
-		-v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \
 		-v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \
 		voicevox/voicevox_engine:build-cpu-ubuntu18.04-latest $(CMD)
 
@@ -195,6 +192,5 @@ build-linux-docker-build-nvidia-ubuntu18.04:
 .PHONY: run-linux-docker-build-nvidia-ubuntu18.04
 run-linux-docker-build-nvidia-ubuntu18.04:
 	docker run --rm -it \
-		-v "$(shell pwd)/cache/Nuitka:/home/user/.cache/Nuitka" \
 		-v "$(shell pwd)/build:/opt/voicevox_engine_build" $(ARGS) \
 		voicevox/voicevox_engine:build-nvidia-ubuntu18.04-latest $(CMD)

README.md

@@ -410,35 +410,18 @@ python make_docs.py
 
 ## ビルド
 
-Build Tools for Visual Studio 2019 が必要です。
+この方法でビルドしたものは、リリースで公開されているものとは異なります。
+また、GPUで利用するにはcuDNNやCUDA、DirectMLなどのライブラリが追加で必要となります。
 
 ```bash
 python -m pip install -r requirements-dev.txt
 
 python generate_licenses.py > licenses.json
 
-python -m nuitka \
-    --standalone \
-    --plugin-enable=numpy \
-    --plugin-enable=multiprocessing \
-    --follow-import-to=numpy \
-    --follow-import-to=aiofiles \
-    --include-package=uvicorn \
-    --include-package=anyio \
-    --include-package-data=pyopenjtalk \
-    --include-package-data=scipy \
-    --include-data-file=licenses.json=./ \
-    --include-data-file=presets.yaml=./ \
-    --include-data-file=default.csv=./ \
-    --include-data-file=engine_manifest.json=./ \
-    --include-data-file=C:/path/to/cuda/*.dll=./ \
-    --include-data-file=C:/path/to/onnxruntime/lib/*.dll=./ \
-    --include-data-dir=.venv/Lib/site-packages/_soundfile_data=./_soundfile_data \
-    --include-data-dir=speaker_info=./speaker_info \
-    --msvc=14.2 \
-    --follow-imports \
-    --no-prefer-source-code \
-    run.py
+# ビルド自体はLIBCORE_PATH及びLIBONNXRUNTIME_PATHの指定がなくても可能です
+LIBCORE_PATH="/path/to/libcore" \
+    LIBONNXRUNTIME_PATH="/path/to/libonnxruntime" \
+    pyinstaller --noconfirm run.spec
 ```
 
 ## 依存関係