v21.7.11

- This is a significant update to how setup work across different platform. It might be causing issues... especially for linux env like runpod. If you encounter problems please report them in the issues so I can try to address them. You can revert to the previous release with `git checkout v21.7.10`

The setup solution is now much more modulat and will simplify requirements support across different environments... hoping this will make it easier to run on different OS.

Author: bmaltais

.gitignore

@@ -19,3 +19,5 @@ test/output
 test/logs
 test/*.json
 test/ft
+requirements_tmp_for_setup.txt
+0.13.3

Dockerfile

@@ -26,8 +26,8 @@ RUN python3 -m pip install wheel
 ## RUN python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers
 
 # Install requirements
-COPY requirements_unix.txt setup.py ./
-RUN python3 -m pip install --use-pep517 -r requirements_unix.txt xformers
+COPY requirements_linux.txt ./setup/setup.py ./
+RUN python3 -m pip install --use-pep517 -r requirements_linux.txt xformers
 
 # Replace pillow with pillow-simd
 RUN python3 -m pip uninstall -y pillow && \

README.md

@@ -102,6 +102,8 @@ If you run on Linux, there is an alternative docker container port with less lim
 
 venv support need to be pre-installed. Can be done on ubuntu 22.04 with `apt install python3.10-venv`
 
+For Linux, make sure to install the cudaNN drivers following the instructions from: `https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64`
+
 Make sure to use a version of python >= 3.10.6 and < 3.11.0
 
 #### Setup
@@ -353,6 +355,10 @@ This will store a backup file with your current locally installed pip packages a
 
 ## Change History
 
+* 2023/06/23 (v21.7.11)
+- This is a significant update to how setup work across different platform. It might be causing issues... especially for linux env like runpod. If you encounter problems please report them in the issues so I can try to address them. You can revert to the previous release with `git checkout v21.7.10`
+
+The setup solution is now much more modulat and will simplify requirements support across different environments... hoping this will make it easier to run on different OS.
 * 2023/06/19 (v21.7.10)
 - Quick fix for linux GUI startup where it would try to install darwin requirements on top of linux. Ugly fix but work. Hopefulle some linux user will improve via a PR.
 * 2023/06/18 (v21.7.9)

gui.bat

@@ -3,16 +3,16 @@
 call .\venv\Scripts\deactivate.bat
 
 :: Calling external python program to check for local modules
-python .\tools\check_local_modules.py --no_question
+python .\setup\check_local_modules.py --no_question
 
 :: Activate the virtual environment
 call .\venv\Scripts\activate.bat
 set PATH=%PATH%;%~dp0venv\Lib\site-packages\torch\lib
 
 :: Validate requirements
-python.exe .\tools\validate_requirements.py
+python.exe .\setup\validate_requirements.py
 
 :: If the exit code is 0, run the kohya_gui.py script with the command-line arguments
 if %errorlevel% equ 0 (
-    python.exe kohya_gui.py %*
+    cmd /k python.exe kohya_gui.py %*
 )

gui.ps1

@@ -29,10 +29,10 @@ if ($pipOutput) {
 $env:PATH += ";$($MyInvocation.MyCommand.Path)\venv\Lib\site-packages\torch\lib"
 
 # Debug info about system
-# python.exe .\tools\debug_info.py
+# python.exe .\setup\debug_info.py
 
 # Validate the requirements and store the exit code
-python.exe .\tools\validate_requirements.py
+python.exe .\setup\validate_requirements.py
 
 # If the exit code is 0, read arguments from gui_parameters.txt (if it exists)
 # and run the kohya_gui.py script with the command-line arguments

gui.sh

@@ -17,6 +17,12 @@ cd "$SCRIPT_DIR"
 source "$SCRIPT_DIR/venv/bin/activate"
 
 # If the requirements are validated, run the kohya_gui.py script with the command-line arguments
-if python  "$SCRIPT_DIR"/tools/validate_requirements_unix.py -r "$SCRIPT_DIR"/requirements_unix.txt; then
-    python "$SCRIPT_DIR/kohya_gui.py" "$@"
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    if python  "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_macos.txt; then
+        python "$SCRIPT_DIR/kohya_gui.py" "$@"
+    fi
+else
+    if python  "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_linux.txt; then
+        python "$SCRIPT_DIR/kohya_gui.py" "$@"
+    fi
 fi

library/convert_model_gui.py

@@ -174,6 +174,10 @@ def gradio_convert_model_tab(headless=False):
         gr.Markdown(
             'This utility can be used to convert from one stable diffusion model format to another.'
         )
+        
+        model_ext = gr.Textbox(value='*.safetensors *.ckpt', visible=False)
+        model_ext_name = gr.Textbox(value='Model types', visible=False)
+        
         with gr.Row():
             source_model_input = gr.Textbox(
                 label='Source model',
@@ -198,7 +202,7 @@ def gradio_convert_model_tab(headless=False):
             )
             button_source_model_file.click(
                 get_file_path,
-                inputs=[source_model_input],
+                inputs=[source_model_input, model_ext, model_ext_name],
                 outputs=source_model_input,
                 show_progress=False,
             )

library/custom_logging.py

@@ -1,6 +1,7 @@
 import os
 import logging
 import time
+import sys
 
 from rich.theme import Theme
 from rich.logging import RichHandler
@@ -23,7 +24,10 @@ def setup_logging(clean=False, debug=False):
     except:
         pass
 
-    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True)
+    if sys.version_info >= (3, 9):
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True)
+    else:
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', force=True)
 
     console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({
         "traceback.border": "black",

pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "library"
+version = "1.0.3"
+description = "Libraries required to run kohya_ss GUI"
+authors = ["Bernard Maltais <bernard@ducourier.com>"]
+license = "Apache-2.0" # Apache Software License
+
+[[tool.poetry.source]]
+name = "library"
+path = "library"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.11"
+
+[tool.poetry.dev-dependencies]

requirements_Ubuntu_20.04.txt requirements.txt

@@ -1,4 +1,3 @@
-accelerate==0.19.0
 albumentations==1.3.0
 altair==4.2.2
 bitsandbytes==0.35.0
@@ -8,25 +7,20 @@ easygui==0.98.3
 einops==0.6.0
 fairscale==0.4.13
 ftfy==6.1.1
-gradio==3.23.0; sys_platform == 'darwin'
-gradio==3.32.0; sys_platform != 'darwin'
-huggingface-hub==0.13.3; sys_platform == 'darwin'
-huggingface-hub==0.13.3; sys_platform != 'darwin'
+gradio==3.33.1
+huggingface-hub>=0.13.3
 lion-pytorch==0.0.6
 lycoris_lora==0.1.6
 opencv-python==4.7.0.68
 prodigyopt==1.0
 pytorch-lightning==1.9.0
 rich==13.4.1
 safetensors==0.2.6
-tensorboard==2.10.1 ; sys_platform != 'darwin'
-tensorboard==2.12.1 ; sys_platform == 'darwin'
-tensorflow==2.10.1; sys_platform != 'darwin'
 timm==0.6.12
 tk==0.1.0
 toml==0.10.2
 transformers==4.26.0
 voluptuous==0.13.1
 wandb==0.15.0
 # for kohya_ss library
-.
+-e . # no_verify leave this to specify not checking this a verification stage

requirements_linux.txt

@@ -0,0 +1,4 @@
+torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # no_verify leave this to specify not checking this a verification stage
+xformers==0.0.20
+accelerate==0.19.0 tensorboard==2.12.1 tensorflow==2.12.0
+-r requirements.txt

requirements_macos_amd64.txt

@@ -0,0 +1,31 @@
+torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+xformers
+accelerate==0.19.0 tensorflow-macos tensorboard==2.12.1
+-r requirements.txt
+
+# accelerate==0.15.0
+# albumentations==1.3.0
+# altair==4.2.2
+# bitsandbytes==0.35.0
+# dadaptation==3.1
+# diffusers[torch]==0.10.2
+# easygui==0.98.3
+# einops==0.6.0
+# fairscale==0.4.13
+# ftfy==6.1.1
+# gradio==3.23.0
+# huggingface-hub==0.13.0
+# lion-pytorch==0.0.6
+# lycoris_lora==0.1.6
+# opencv-python==4.7.0.68
+# pytorch-lightning==1.9.0
+# rich==13.4.1
+# safetensors==0.2.6
+# timm==0.6.12
+# tk==0.1.0
+# toml==0.10.2
+# transformers==4.26.0
+# voluptuous==0.13.1
+# wandb==0.15.0
+# # for kohya_ss library
+# .

requirements_macos_arm64.txt

@@ -0,0 +1,31 @@
+torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+xformers
+accelerate==0.19.0 tensorflow-metal tensorboard==2.12.1
+-r requirements.txt
+
+# accelerate==0.15.0
+# albumentations==1.3.0
+# altair==4.2.2
+# bitsandbytes==0.35.0
+# dadaptation==3.1
+# diffusers[torch]==0.10.2
+# easygui==0.98.3
+# einops==0.6.0
+# fairscale==0.4.13
+# ftfy==6.1.1
+# gradio==3.23.0
+# huggingface-hub==0.13.0
+# lion-pytorch==0.0.6
+# lycoris_lora==0.1.6
+# opencv-python==4.7.0.68
+# pytorch-lightning==1.9.0
+# rich==13.4.1
+# safetensors==0.2.6
+# timm==0.6.12
+# tk==0.1.0
+# toml==0.10.2
+# transformers==4.26.0
+# voluptuous==0.13.1
+# wandb==0.15.0
+# # for kohya_ss library
+# .

requirements_unix.txt

@@ -1,29 +1,31 @@
-accelerate==0.15.0
-albumentations==1.3.0
-altair==4.2.2
-bitsandbytes==0.35.0
-dadaptation==3.1
-diffusers[torch]==0.10.2
-easygui==0.98.3
-einops==0.6.0
-fairscale==0.4.13
-ftfy==6.1.1
-gradio==3.32.0
-huggingface-hub==0.13.3
-lion-pytorch==0.0.6
-lycoris_lora==0.1.6
-opencv-python==4.7.0.68
-prodigyopt==1.0
-pytorch-lightning==1.9.0
-rich==13.4.1
-safetensors==0.2.6
-tensorboard==2.10.1
-tensorflow==2.10.1
-timm==0.6.12
-tk==0.1.0
-toml==0.10.2
-transformers==4.26.0
-voluptuous==0.13.1
-wandb==0.15.0
-# for kohya_ss library
-.
+torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116 # no_verify
+https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps # no_verify
+accelerate==0.15.0 tensorboard==2.10.1 tensorflow==2.10.1
+-r requirements.txt
+
+# albumentations==1.3.0
+# altair==4.2.2
+# bitsandbytes==0.35.0
+# dadaptation==3.1
+# diffusers[torch]==0.10.2
+# easygui==0.98.3
+# einops==0.6.0
+# fairscale==0.4.13
+# ftfy==6.1.1
+# gradio==3.32.0
+# huggingface-hub==0.13.3
+# lion-pytorch==0.0.6
+# lycoris_lora==0.1.6
+# opencv-python==4.7.0.68
+# prodigyopt==1.0
+# pytorch-lightning==1.9.0
+# rich==13.4.1
+# safetensors==0.2.6
+# timm==0.6.12
+# tk==0.1.0
+# toml==0.10.2
+# transformers==4.26.0
+# voluptuous==0.13.1
+# wandb==0.15.0
+# # for kohya_ss library
+# .