Refactor setup (bmaltais#1030)

* Refactor setup in setup folder

* More refactoring

* Remove need to run setup.sh as sudo

* Fix for unix undel python 3.8

* Create setup_common.py

* Fix windows setup

* Refining setup

---------

Co-authored-by: Your Name <you@example.com>

.gitignore

@@ -19,3 +19,4 @@ test/output
 test/logs
 test/*.json
 test/ft
+requirements_tmp_for_setup.txt

Dockerfile

@@ -26,8 +26,8 @@ RUN python3 -m pip install wheel
 ## RUN python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers
 
 # Install requirements
-COPY requirements_unix.txt setup.py ./
-RUN python3 -m pip install --use-pep517 -r requirements_unix.txt xformers
+COPY requirements_linux.txt ./setup/setup.py ./
+RUN python3 -m pip install --use-pep517 -r requirements_linux.txt xformers
 
 # Replace pillow with pillow-simd
 RUN python3 -m pip uninstall -y pillow && \

README.md

@@ -102,6 +102,8 @@ If you run on Linux, there is an alternative docker container port with less lim
 
 venv support need to be pre-installed. Can be done on ubuntu 22.04 with `apt install python3.10-venv`
 
+For Linux, make sure to install the cudaNN drivers following the instructions from: `https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64`
+
 Make sure to use a version of python >= 3.10.6 and < 3.11.0
 
 #### Setup

gui.bat

@@ -3,14 +3,14 @@
 call .\venv\Scripts\deactivate.bat
 
 :: Calling external python program to check for local modules
-python .\tools\check_local_modules.py --no_question
+python .\setup\check_local_modules.py --no_question
 
 :: Activate the virtual environment
 call .\venv\Scripts\activate.bat
 set PATH=%PATH%;%~dp0venv\Lib\site-packages\torch\lib
 
 :: Validate requirements
-python.exe .\tools\validate_requirements.py
+python.exe .\setup\validate_requirements.py
 
 :: If the exit code is 0, run the kohya_gui.py script with the command-line arguments
 if %errorlevel% equ 0 (

gui.ps1

@@ -29,10 +29,10 @@ if ($pipOutput) {
 $env:PATH += ";$($MyInvocation.MyCommand.Path)\venv\Lib\site-packages\torch\lib"
 
 # Debug info about system
-# python.exe .\tools\debug_info.py
+# python.exe .\setup\debug_info.py
 
 # Validate the requirements and store the exit code
-python.exe .\tools\validate_requirements.py
+python.exe .\setup\validate_requirements.py
 
 # If the exit code is 0, read arguments from gui_parameters.txt (if it exists)
 # and run the kohya_gui.py script with the command-line arguments

gui.sh

@@ -17,6 +17,12 @@ cd "$SCRIPT_DIR"
 source "$SCRIPT_DIR/venv/bin/activate"
 
 # If the requirements are validated, run the kohya_gui.py script with the command-line arguments
-if python  "$SCRIPT_DIR"/tools/validate_requirements_unix.py -r "$SCRIPT_DIR"/requirements_unix.txt; then
-    python "$SCRIPT_DIR/kohya_gui.py" "$@"
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    if python  "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_macos.txt; then
+        python "$SCRIPT_DIR/kohya_gui.py" "$@"
+    fi
+else
+    if python  "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_linux.txt; then
+        python "$SCRIPT_DIR/kohya_gui.py" "$@"
+    fi
 fi

library/custom_logging.py

@@ -1,6 +1,7 @@
 import os
 import logging
 import time
+import sys
 
 from rich.theme import Theme
 from rich.logging import RichHandler
@@ -23,7 +24,10 @@ def setup_logging(clean=False, debug=False):
     except:
         pass
 
-    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True)
+    if sys.version_info >= (3, 9):
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True)
+    else:
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', force=True)
 
     console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({
         "traceback.border": "black",

pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kohya_ss"
+version = "1.0.3"
+description = "A GUI wrapper for kohya-ss SD scipts enabling LoRA training with an easy-to-use web application."
+authors = [
+    {name = "bmaltais", email = "bernard@ducourier.com"},
+]
+readme = "README.md"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+]
+
+[tool.poetry]
+license = "Apache-2.0"
+
+[tool.setuptools.packages.find]
+where = ["library"] # We have to explicitly tell build tools where to look

requirements_Ubuntu_20.04.txt → requirements_linux.txt

@@ -8,20 +8,17 @@ easygui==0.98.3
 einops==0.6.0
 fairscale==0.4.13
 ftfy==6.1.1
-gradio==3.23.0; sys_platform == 'darwin'
-gradio==3.32.0; sys_platform != 'darwin'
-huggingface-hub==0.13.3; sys_platform == 'darwin'
-huggingface-hub==0.13.3; sys_platform != 'darwin'
+gradio==3.32.0
+huggingface-hub==0.13.3
 lion-pytorch==0.0.6
 lycoris_lora==0.1.6
 opencv-python==4.7.0.68
 prodigyopt==1.0
 pytorch-lightning==1.9.0
 rich==13.4.1
 safetensors==0.2.6
-tensorboard==2.10.1 ; sys_platform != 'darwin'
-tensorboard==2.12.1 ; sys_platform == 'darwin'
-tensorflow==2.10.1; sys_platform != 'darwin'
+tensorboard==2.12.1
+tensorflow==2.12.0
 timm==0.6.12
 tk==0.1.0
 toml==0.10.2

requirements_unix.txt → requirements_macos.txt

@@ -8,19 +8,15 @@ easygui==0.98.3
 einops==0.6.0
 fairscale==0.4.13
 ftfy==6.1.1
-gradio==3.23.0; sys_platform == 'darwin'
-gradio==3.32.0; sys_platform != 'darwin'
-huggingface-hub==0.13.0; sys_platform == 'darwin'
-huggingface-hub==0.13.3; sys_platform != 'darwin'
+gradio==3.23.0
+huggingface-hub==0.13.0
 lion-pytorch==0.0.6
 lycoris_lora==0.1.6
 opencv-python==4.7.0.68
 pytorch-lightning==1.9.0
 rich==13.4.1
 safetensors==0.2.6
-tensorboard==2.10.1 ; sys_platform != 'darwin'
-tensorboard==2.12.1 ; sys_platform == 'darwin'
-tensorflow==2.10.1; sys_platform != 'darwin'
+tensorboard==2.12.1
 timm==0.6.12
 tk==0.1.0
 toml==0.10.2

setup.bat

@@ -20,11 +20,11 @@ mkdir ".\logs\setup" > nul 2>&1
 call .\venv\Scripts\deactivate.bat
 
 :: Calling external python program to check for local modules
-python .\tools\check_local_modules.py
+python .\setup\check_local_modules.py
 
 call .\venv\Scripts\activate.bat
 
-python .\tools\setup_windows.py
+python .\setup\setup_windows.py
 
 :: Deactivate the virtual environment
 call .\venv\Scripts\deactivate.bat

setup.ps1

@@ -17,11 +17,11 @@ $null = New-Item -ItemType Directory -Force -Path ".\logs\setup"
 & .\venv\Scripts\deactivate.bat
 
 # Calling external python program to check for local modules
-& .\venv\Scripts\python.exe .\tools\check_local_modules.py
+& .\venv\Scripts\python.exe .\setup\check_local_modules.py
 
 & .\venv\Scripts\activate.bat
 
-& .\venv\Scripts\python.exe .\tools\setup_windows.py
+& .\venv\Scripts\python.exe .\setup\setup_windows.py
 
 # Deactivate the virtual environment
 & .\venv\Scripts\deactivate.bat